PT-2375 - fixed pt-table-sync for tables with generated columns

Previously, pt-table-sync generated DML statements that included
the generated columns of a table, which is however rejected by
the database and not necessary for the required sync statements.

Author: hpoettker

bin/pt-table-sync

@@ -3708,6 +3708,7 @@ sub sort_cols {
    my @cols;
    if ( $self->{tbl_struct} ) {
       my $pos = $self->{tbl_struct}->{col_posn};
+      my $is_generated = $self->{tbl_struct}->{is_generated};
       my @not_in_tbl;
       @cols = sort {
             $pos->{$a} <=> $pos->{$b}
@@ -3721,6 +3722,9 @@ sub sort_cols {
                1;
             }
          }
+         grep {
+            !$is_generated->{$_}
+         }
          sort keys %$row;
       push @cols, @not_in_tbl if @not_in_tbl;
    }

lib/ChangeHandler.pm

@@ -462,7 +462,8 @@ sub get_changes {
 
 
 # Sub: sort_cols
-#   Sort a row's columns based on their real order in the table.
+#   Sort a row's columns based on their real order in the table, and remove
+#   generated columns.
 #   This requires that the optional tbl_struct arg was passed to <new()>.
 #   If not, the rows are sorted alphabetically.
 #
@@ -476,6 +477,7 @@ sub sort_cols {
    my @cols;
    if ( $self->{tbl_struct} ) {
       my $pos = $self->{tbl_struct}->{col_posn};
+      my $is_generated = $self->{tbl_struct}->{is_generated};
       my @not_in_tbl;
       @cols = sort {
             $pos->{$a} <=> $pos->{$b}
@@ -489,6 +491,9 @@ sub sort_cols {
                1;
             }
          }
+         grep {
+            !$is_generated->{$_}
+         }
          sort keys %$row;
       push @cols, @not_in_tbl if @not_in_tbl;
    }

t/lib/ChangeHandler.t

@@ -339,6 +339,85 @@ SKIP: {
    );
 };
 
+# #############################################################################
+# PT-2375: pt-table-sync must handle generated columns correctly
+# #############################################################################
+$row = {
+   id  => 1,
+   foo => 'foo',
+   bar => 'bar',
+};
+$tbl_struct = {
+   col_posn => { id=>0, foo=>1, bar=>2 },
+   is_generated => {foo=>1}
+};
+$ch = new ChangeHandler(
+   Quoter     => $q,
+   right_db   => 'test',       # dst
+   right_tbl  => 'pt-2375',
+   left_db    => 'test',       # src
+   left_tbl   => 'pt-2375',
+   actions    => [ sub { push @rows, @_ } ],
+   replace    => 0,
+   queue      => 0,
+   tbl_struct => $tbl_struct,
+);
+
+@rows = ();
+@dbhs = ();
+
+is(
+   $ch->make_INSERT($row, [qw(id foo bar)]),
+   "INSERT INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'bar')",
+   'make_INSERT() omits generated columns'
+);
+
+is(
+   $ch->make_REPLACE($row, [qw(id foo bar)]),
+   "REPLACE INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'bar')",
+   'make_REPLACE() omits generated columns'
+);
+
+is(
+   $ch->make_UPDATE($row, [qw(id foo)]),
+   "UPDATE `test`.`pt-2375` SET `bar`='bar' WHERE `id`='1' AND `foo`='foo' LIMIT 1",
+   'make_UPDATE() omits generated columns from SET phrase but includes in WHERE phrase'
+);
+
+is(
+   $ch->make_DELETE($row, [qw(id foo bar)]),
+   "DELETE FROM `test`.`pt-2375` WHERE `id`='1' AND `foo`='foo' AND `bar`='bar' LIMIT 1",
+   'make_DELETE() includes generated columns in WHERE phrase'
+);
+
+SKIP: {
+   skip 'Cannot connect to sandbox master', 3 unless $master_dbh;
+
+   $master_dbh->do('DROP TABLE IF EXISTS test.`pt-2375`');
+   $master_dbh->do('CREATE TABLE test.`pt-2375` (id INT, foo varchar(16) as ("foo"), bar char)');
+   $master_dbh->do("INSERT INTO test.`pt-2375` (`id`, `bar`) VALUES (1,'a'),(2,'b')");
+
+   $ch->fetch_back($master_dbh);
+
+   is(
+      $ch->make_INSERT($row, [qw(id foo)]),
+      "INSERT INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'a')",
+      'make_INSERT() omits generated columns, with fetch-back'
+   );
+
+   is(
+      $ch->make_REPLACE($row, [qw(id foo)]),
+      "REPLACE INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'a')",
+      'make_REPLACE() omits generated columns, with fetch-back'
+   );
+
+   is(
+      $ch->make_UPDATE($row, [qw(id foo)]),
+      "UPDATE `test`.`pt-2375` SET `bar`='a' WHERE `id`='1' AND `foo`='foo' LIMIT 1",
+      'make_UPDATE() omits generated columns from SET phrase, with fetch-back'
+   );
+};
+
 # #############################################################################
 # Issue 641: Make mk-table-sync use hex for binary/blob data
 # #############################################################################

t/pt-table-sync/pt-2375.t

@@ -0,0 +1,80 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-table-sync";
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica1_dbh = $sb->get_dbh_for('replica1');
+
+if ( !$source_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+elsif ( !$replica1_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox replica1';
+}
+else {
+   plan tests => 3;
+}
+
+my $output;
+
+# #############################################################################
+# Test generated REPLACE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2375.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("delete from `test`.`test_table` where `id`=1");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main('--sync-to-source',
+      'h=127.0.0.1,P=12346,u=msandbox,p=msandbox',
+      qw(-t test.test_table --print --execute))
+   },
+));
+chomp($output);
+is(
+   $output,
+   "REPLACE INTO `test`.`test_table`(`id`, `value`) VALUES ('1', '24');",
+   "Generated columns are not used in REPLACE statements"
+);
+
+# #############################################################################
+# Test generated UPDATE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2375.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("update `test`.`test_table` set `value`=55 where `id`=2");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main(qw(--print --execute),
+      "h=127.0.0.1,P=12346,u=msandbox,p=msandbox,D=test,t=test_table",
+      "h=127.0.0.1,P=12345,u=msandbox,p=msandbox,D=test,t=test_table");
+   }
+));
+chomp($output);
+is(
+   $output,
+   "UPDATE `test`.`test_table` SET `value`='55' WHERE `id`='2' LIMIT 1;",
+   "Generated columns are not used in UPDATE statements"
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-table-sync/samples/pt-2375.sql

@@ -0,0 +1,12 @@
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE test;
+USE test;
+
+CREATE TABLE `test_table` (
+  `id` INT AUTO_INCREMENT PRIMARY KEY,
+  `value` INT NOT NULL,
+  `derived_value` INT AS (2*`value`)
+) ENGINE=InnoDB;
+
+INSERT INTO `test_table` (`value`) VALUES (24);
+INSERT INTO `test_table` (`value`) VALUES (42);