From 23bb56130a44517859c1a702a7cb12fad60e06e2 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Fri, 6 Feb 2026 19:04:18 +0900 Subject: [PATCH 01/10] Add bind_param_attributes to ::DBD::MariaDB --- lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm b/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm index cec472b..03ac1db 100644 --- a/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm +++ b/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm @@ -7,4 +7,16 @@ use base qw( Data::ObjectDriver::Driver::DBD::mysql ); sub fetch_id { $_[3]->{mariadb_insertid} || $_[3]->{insertid} } +sub bind_param_attributes { + my ($dbd, $data_type) = @_; + if ($data_type) { + if ($data_type eq 'blob') { + return DBI::SQL_BINARY; + } elsif ($data_type eq 'binchar') { + return DBI::SQL_BINARY; + } + } + return; +} + 1; From 4e2e585e7d08d10727f7c8d52d0aaed3da62d057 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Fri, 6 Feb 2026 19:04:30 +0900 Subject: [PATCH 02/10] Add a test --- t/10-resultset-blob.t | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 t/10-resultset-blob.t diff --git a/t/10-resultset-blob.t b/t/10-resultset-blob.t new file mode 100644 index 0000000..a0776c8 --- /dev/null +++ b/t/10-resultset-blob.t @@ -0,0 +1,46 @@ +# $Id: 01-col-inheritance.t 989 2005-09-23 19:58:01Z btrott $ + +use strict; +use warnings; + +use lib 't/lib'; + +$Data::ObjectDriver::DEBUG = 0; +use Test::More; +use DodTestUtil; +BEGIN { eval { require Crypt::URandom; 1 } or plan skip_all => 'requires Crypt::URandom' } + +BEGIN { DodTestUtil->check_driver } + +plan tests => 3; + +setup_dbs({ + global => [ qw( wines ) ], +}); + +use Wine; +use Storable; + +my $wine = Wine->new; +$wine->name("Saumur Champigny, Le Grand Clos 2001"); +$wine->rating(4); + +## generate some binary data (SQL_BLOB / MEDIUMBLOB) +my $binary = Crypt::URandom::urandom(300); +$wine->content($binary); +ok($wine->save, 'Object saved successfully'); + +my $iter; + +$iter = Data::ObjectDriver::Iterator->new(sub {}); +my $wine_id = $wine->id; +undef $wine; +$wine = Wine->lookup($wine_id); + +ok $wine; +ok $wine->content eq $binary; + +# TODO: bulk_insert doesn't support blob yet. We need to change some of its API so that we can call column_def in each dbd's bulk_insert + +disconnect_all($wine); +teardown_dbs(qw( global )); From f5bc24b421601be0eeac49117193273c9b0ee3cf Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Fri, 6 Feb 2026 19:05:04 +0900 Subject: [PATCH 03/10] Add Crypt::URandom to cpanfile for testing --- cpanfile | 1 + 1 file changed, 1 insertion(+) diff --git a/cpanfile b/cpanfile index f4639d0..18f6ed9 100644 --- a/cpanfile +++ b/cpanfile @@ -23,6 +23,7 @@ on develop => sub { on test => sub { requires 'version'; requires 'Tie::IxHash'; + requires 'Crypt::URandom'; }; feature 'test_sqlite', 'Test SQLite' => sub { From 709edda622d1d35c8b531858cb86762836dab72b Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 10 Feb 2026 09:33:17 +0900 Subject: [PATCH 04/10] Better to use a core module --- cpanfile | 2 +- t/10-resultset-blob.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpanfile b/cpanfile index 18f6ed9..3ea7334 100644 --- a/cpanfile +++ b/cpanfile @@ -23,7 +23,7 @@ on develop => sub { on test => sub { requires 'version'; requires 'Tie::IxHash'; - requires 'Crypt::URandom'; + requires 'Digest::SHA'; }; feature 'test_sqlite', 'Test SQLite' => sub { diff --git a/t/10-resultset-blob.t b/t/10-resultset-blob.t index a0776c8..ff1039e 100644 --- a/t/10-resultset-blob.t +++ b/t/10-resultset-blob.t @@ -8,7 +8,7 @@ use lib 't/lib'; $Data::ObjectDriver::DEBUG = 0; use Test::More; use DodTestUtil; -BEGIN { eval { require Crypt::URandom; 1 } or plan skip_all => 'requires Crypt::URandom' } +BEGIN { eval { require Digest::SHA; 1 } or plan skip_all => 'requires Digest::SHA' } BEGIN { DodTestUtil->check_driver } @@ -26,7 +26,7 @@ $wine->name("Saumur Champigny, Le Grand Clos 2001"); $wine->rating(4); ## generate some binary data (SQL_BLOB / MEDIUMBLOB) -my $binary = Crypt::URandom::urandom(300); +my $binary = Digest::SHA::sha1("binary"); $wine->content($binary); ok($wine->save, 'Object saved successfully'); From 90ef1082a621cba42019a1bc9f978cded2e988c8 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 10 Feb 2026 22:43:13 +0900 Subject: [PATCH 05/10] Pass a hashref of attributes that need special care as the last parameter to bulk_insert of the dbd --- lib/Data/ObjectDriver/Driver/DBI.pm | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/Data/ObjectDriver/Driver/DBI.pm b/lib/Data/ObjectDriver/Driver/DBI.pm index 449f6eb..ed84c50 100644 --- a/lib/Data/ObjectDriver/Driver/DBI.pm +++ b/lib/Data/ObjectDriver/Driver/DBI.pm @@ -644,7 +644,15 @@ sub bulk_insert { my $tbl = $driver->table_for($class); my @db_cols = map {$dbd->db_column_name($tbl, $_) } @{$cols}; - return $dbd->bulk_insert($dbh, $tbl, \@db_cols, $data); + my %attrs; + my $col_defs = $class->properties->{column_defs}; + for my $col (@$cols) { + my $type = $col_defs->{$col} || 'char'; + my $attr = $dbd->bind_param_attributes($type) or next; + $attrs{$col} = $attr; + } + + return $dbd->bulk_insert($dbh, $tbl, \@db_cols, $data, \%attrs); } sub begin_work { From 937504f8aaab8826c4d8ecdd51c87ddb5e9f68e4 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 10 Feb 2026 22:43:45 +0900 Subject: [PATCH 06/10] Add customized bulk_insert to ::DBD::MariaDB --- lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm b/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm index 03ac1db..f01fac3 100644 --- a/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm +++ b/lib/Data/ObjectDriver/Driver/DBD/MariaDB.pm @@ -3,6 +3,7 @@ package Data::ObjectDriver::Driver::DBD::MariaDB; use strict; use warnings; +use Carp; use base qw( Data::ObjectDriver::Driver::DBD::mysql ); sub fetch_id { $_[3]->{mariadb_insertid} || $_[3]->{insertid} } @@ -19,4 +20,42 @@ sub bind_param_attributes { return; } +sub bulk_insert { + my $dbd = shift; + my $dbh = shift; + my $table = shift; + + my $cols = shift; + my $rows_ref = shift; + my $attrs = shift || {}; + + croak "Usage bulk_insert(dbd, dbh, table, columnref, rowsref)" + unless (defined $dbd && defined $dbh && defined $table && defined $cols && + defined $rows_ref); + + return 0e0 if (scalar(@{$rows_ref}) == 0); + + my $sql = "INSERT INTO $table (" . join(',', @{$cols}) . ") VALUES\n"; + + my $one_data_row = "(" . (join ',', (('?') x @$cols)) . ")"; + my $ph = join ",", (($one_data_row) x @$rows_ref); + $sql .= $ph; + + # For now just write all data, at some point we need to lookup the + # maximum packet size for SQL + + if (%$attrs) { + my $sth = $dbh->prepare($sql); + my $i = 1; + for my $row (@$rows_ref) { + for (my $j = 0; $j < @$cols; $j++) { + $sth->bind_param($i++, $row->[$j], $attrs->{$cols->[$j]}); + } + } + $sth->execute; + } else { + return $dbh->do($sql, undef, map { @$_ } @$rows_ref); + } +} + 1; From 58eac4e465a177bb40e4d0bf8a5c59cec25a26ff Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 10 Feb 2026 22:44:56 +0900 Subject: [PATCH 07/10] Tweak ::DBD::Pg::bulk_insert not to use COPY statement if some of the attributes require binary handling --- lib/Data/ObjectDriver/Driver/DBD/Pg.pm | 34 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/Data/ObjectDriver/Driver/DBD/Pg.pm b/lib/Data/ObjectDriver/Driver/DBD/Pg.pm index 86ad114..5dc1392 100644 --- a/lib/Data/ObjectDriver/Driver/DBD/Pg.pm +++ b/lib/Data/ObjectDriver/Driver/DBD/Pg.pm @@ -51,15 +51,33 @@ sub bulk_insert { my $cols = shift; my $rows_ref = shift; - - my $sql = "COPY $table (" . join(',', @{$cols}) . ') from stdin'; - - $dbh->do($sql); - foreach my $row (@{$rows_ref}) { - my $line = join("\t", map {$_ || '\N'} @{$row}); - $dbh->pg_putline("$line\n"); + my $attrs = shift || {}; + + if (%$attrs) { + my $sql = "INSERT INTO $table (" . join(',', @{$cols}) . ") VALUES\n"; + + my $one_data_row = "(" . (join ',', (('?') x @$cols)) . ")"; + my $ph = join ",", (($one_data_row) x @$rows_ref); + $sql .= $ph; + + my $sth = $dbh->prepare($sql); + my $i = 1; + for my $row (@$rows_ref) { + for (my $j = 0; $j < @$cols; $j++) { + $sth->bind_param($i++, $row->[$j], $attrs->{$cols->[$j]}); + } + } + $sth->execute; + } else { + my $sql = "COPY $table (" . join(',', @{$cols}) . ') from stdin'; + + $dbh->do($sql); + foreach my $row (@{$rows_ref}) { + my $line = join("\t", map {$_ || '\N'} @{$row}); + $dbh->pg_putline("$line\n"); + } + return $dbh->pg_endcopy(); } - return $dbh->pg_endcopy(); } sub map_error_code { From 8ee9c84971419bcb066bdc8ca8a72a9df19fb9e2 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 10 Feb 2026 22:45:28 +0900 Subject: [PATCH 08/10] Tweak bulk_insert for SQLite and Oracle --- lib/Data/ObjectDriver/Driver/DBD/Oracle.pm | 7 ++++++- lib/Data/ObjectDriver/Driver/DBD/SQLite.pm | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/Data/ObjectDriver/Driver/DBD/Oracle.pm b/lib/Data/ObjectDriver/Driver/DBD/Oracle.pm index 5020193..b1f1bf5 100644 --- a/lib/Data/ObjectDriver/Driver/DBD/Oracle.pm +++ b/lib/Data/ObjectDriver/Driver/DBD/Oracle.pm @@ -68,6 +68,7 @@ sub bulk_insert { my $table = shift; my $cols = shift; my $rows_ref = shift; + my $attrs = shift || {}; my $sql = "INSERT INTO $table(" . join(',', @$cols) @@ -76,7 +77,11 @@ sub bulk_insert { . ")"; my $sth = $dbh->prepare($sql); foreach my $row (@{ $rows_ref || []}) { - $sth->execute(@$row); + my $i = 1; + for (my $j = 0; $j < @$cols; $j++) { + $sth->bind_param($i++, $row->[$j], $attrs->{$cols->[$j]}); + } + $sth->execute; } return 1; } diff --git a/lib/Data/ObjectDriver/Driver/DBD/SQLite.pm b/lib/Data/ObjectDriver/Driver/DBD/SQLite.pm index de65575..18840f2 100644 --- a/lib/Data/ObjectDriver/Driver/DBD/SQLite.pm +++ b/lib/Data/ObjectDriver/Driver/DBD/SQLite.pm @@ -46,13 +46,18 @@ sub bulk_insert { my $cols = shift; my $rows_ref = shift; + my $attrs = shift || {}; my $sql = "INSERT INTO $table(" . join(',', @{$cols}) . ") VALUES (" . join(',', map {'?'} @{$cols}) . ")\n"; my $sth = $dbh->prepare($sql); foreach my $row (@{$rows_ref}) { - $sth->execute(@{$row}); + my $i = 1; + for (my $j = 0; $j < @$cols; $j++) { + $sth->bind_param($i++, $row->[$j], $attrs->{$cols->[$j]}); + } + $sth->execute; } # For now just write all data, at some point we need to lookup the From 60da6c6faa47a6642105992adaf0e2893c08caf4 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 10 Feb 2026 22:46:01 +0900 Subject: [PATCH 09/10] Add tests for bulk insertion of binary --- t/10-resultset-blob.t | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/t/10-resultset-blob.t b/t/10-resultset-blob.t index ff1039e..1ced70f 100644 --- a/t/10-resultset-blob.t +++ b/t/10-resultset-blob.t @@ -12,7 +12,7 @@ BEGIN { eval { require Digest::SHA; 1 } or plan skip_all => 'requires Digest::SH BEGIN { DodTestUtil->check_driver } -plan tests => 3; +plan tests => 5; setup_dbs({ global => [ qw( wines ) ], @@ -40,7 +40,13 @@ $wine = Wine->lookup($wine_id); ok $wine; ok $wine->content eq $binary; -# TODO: bulk_insert doesn't support blob yet. We need to change some of its API so that we can call column_def in each dbd's bulk_insert +my @names = qw(Margaux Latour); +Wine->bulk_insert([qw(name content)], [ map {[$_, Digest::SHA::sha1($_)]} @names ]); + +for my $name (@names) { + my ($found) = Wine->search({name => $name}); + ok $found->content eq Digest::SHA::sha1($name); +} disconnect_all($wine); teardown_dbs(qw( global )); From e0b218443715ac4740facad0b80c191a8247a9ed Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Wed, 11 Feb 2026 00:36:45 +0900 Subject: [PATCH 10/10] Better to use db_col --- lib/Data/ObjectDriver/Driver/DBI.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/Data/ObjectDriver/Driver/DBI.pm b/lib/Data/ObjectDriver/Driver/DBI.pm index ed84c50..c722f17 100644 --- a/lib/Data/ObjectDriver/Driver/DBI.pm +++ b/lib/Data/ObjectDriver/Driver/DBI.pm @@ -642,14 +642,16 @@ sub bulk_insert { # pass this directly to the backend DBD my $dbh = $driver->rw_handle($class->properties->{db}); my $tbl = $driver->table_for($class); - my @db_cols = map {$dbd->db_column_name($tbl, $_) } @{$cols}; + my @db_cols; my %attrs; my $col_defs = $class->properties->{column_defs}; for my $col (@$cols) { + my $db_col = $dbd->db_column_name($tbl, $col); + push @db_cols, $db_col; my $type = $col_defs->{$col} || 'char'; my $attr = $dbd->bind_param_attributes($type) or next; - $attrs{$col} = $attr; + $attrs{$db_col} = $attr; } return $dbd->bulk_insert($dbh, $tbl, \@db_cols, $data, \%attrs);