Leverage binary-upgrade facility of pg_dump

While adding a new coordinator or a datanode, we take a schema dump of an
existing node and restore it on the new node. This has some problems,
especially while restoring tables with dropped columns and different column
ordering. The new node won't restore the dropped columns and may recreate
columns in different ordering, for inheritted tables for example. This then
leads to issues during query execution.

We now leverage binary-upgrade dump facility available in pg_dump/pg_restore.
While XL is not very sensitive to OID preservation across nodes, we do care
about column ordering. So binary-upgrade's OID preservation facility is of not
great interest to us, but it shouldn't harm also.

Also ensure that while restoring sequence states, we don't force update the GTM
state.

A tap test is added to check the node addition this way.

The original report came from Krzysztof Nienartowicz. Patch and further work by
me.

diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c
index caa5cba2b8b5927702bc391589b3959ee31b560e..1c8bcf2b73b0ab0920303b2754b064f473810fd2 100644 (file)
--- a/src/backend/commands/sequence.c
+++ b/src/backend/commands/sequence.c
@@ -1060,6 +1060,11 @@ do_setval(Oid relid, int64 next, bool iscalled)
                                                bufm, bufx)));
        }
 
+       /*
+        * During adding a new node, don't update the GTM state since it may have
+        * already advanced.
+        */
+       if (!isRestoreMode)
        {
                char *seqname = GetGlobalSeqName(seqrel, NULL, NULL);
 

diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 58608795a90ecef515c061ebfd1947fbc03df063..50404cc044aaab3cf2ebc0b730eeda3540fde705 100644 (file)
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -11544,8 +11544,10 @@ MergeAttributesIntoExisting(Relation child_rel, Relation parent_rel)
                         * In common cases of partitioning, the parent table and the
                         * partition tables will be created at the very beginning and if
                         * altered, they will be altered together.
+                        *
+                        * Make exception while restoring a schema during node addition.
                         */
-                       if (attribute->attnum != childatt->attnum)
+                       if (!isRestoreMode && attribute->attnum != childatt->attnum)
                                ereport(ERROR,
                                                (errcode(ERRCODE_DATATYPE_MISMATCH),
                                                 errmsg("table \"%s\" contains column \"%s\" at "

diff --git a/src/bin/pgxc_ctl/coord_cmd.c b/src/bin/pgxc_ctl/coord_cmd.c
index 52715672fdd81812faadd04a3487af762b22a5e7..bbbbc9d47a8230ee9586059f714c59a872248822 100644 (file)
--- a/src/bin/pgxc_ctl/coord_cmd.c
+++ b/src/bin/pgxc_ctl/coord_cmd.c
@@ -1183,14 +1183,24 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
        fprintf(lockf, "select pgxc_lock_for_backup();\n");     /* Keep open until the end of the addition. */
        fflush(lockf);
 
-       /* pg_dumpall */
+       /*
+        * pg_dumpall
+        *
+        * We leverage facilities provides by binary-upgrade to ensure a consistent
+        * column ordering between other nodes in the cluster. Essentially, it will
+        * mimic dropped columns by adding dummy columns and later dropping those
+        * to arrive at a final consistent ordering.
+        *
+        * The server must then be started in binary-upgrade mode (-b option) while
+        * restoring the dump.
+        */
        createLocalFileName(GENERAL, pgdumpall_out, MAXPATH);
-       doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes --file=%s",
+       doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes --binary-upgrade --file=%s",
                                   aval(VAR_coordPorts)[connCordIndx],
                                   aval(VAR_coordMasterServers)[connCordIndx], pgdumpall_out);
 
-       /* Start the new coordinator */
-       doImmediate(host, NULL, "pg_ctl start -w -Z restoremode -D %s -o -i", dir);
+       /* Start the new coordinator, in binary upgrade mode */
+       doImmediate(host, NULL, "pg_ctl start -w -Z restoremode -D %s -o '-i -b'", dir);
 
        /* Allow the new coordinator to start up by sleeping for a couple of seconds */
        pg_usleep(2000000L);

diff --git a/src/bin/pgxc_ctl/datanode_cmd.c b/src/bin/pgxc_ctl/datanode_cmd.c
index 811c7388ca1b8c38d122c61c2cebf09013d89cbf..f43ea8bdb0dd30abda8554ffb7412215e64581c0 100644 (file)
--- a/src/bin/pgxc_ctl/datanode_cmd.c
+++ b/src/bin/pgxc_ctl/datanode_cmd.c
@@ -1222,15 +1222,25 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
        fprintf(lockf, "select pgxc_lock_for_backup();\n");     /* Keep open until the end of the addition. */
        fflush(lockf);
 
-       /* pg_dumpall */
+       /*
+        * pg_dumpall
+        *
+        * We leverage facilities provides by binary-upgrade to ensure a consistent
+        * column ordering between other nodes in the cluster. Essentially, it will
+        * mimic dropped columns by adding dummy columns and later dropping those
+        * to arrive at a final consistent ordering.
+        *
+        * The server must then be started in binary-upgrade mode (-b option) while
+        * restoring the dump.
+        */
        createLocalFileName(GENERAL, pgdumpall_out, MAXPATH);
        if (restore_dnode_idx != -1)
-               doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes >%s",
+               doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --binary-upgrade --dump-nodes >%s",
                                   aval(VAR_datanodePorts)[restore_dnode_idx],
                                   aval(VAR_datanodeMasterServers)[restore_dnode_idx],
                                   pgdumpall_out);
        else if (restore_coord_idx != -1)
-               doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes >%s",
+               doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --binary-upgrade --dump-nodes >%s",
                                           aval(VAR_coordPorts)[restore_coord_idx],
                                           aval(VAR_coordMasterServers)[restore_coord_idx],
                                           pgdumpall_out);
@@ -1240,8 +1250,8 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
                return 1;
        }
 
-       /* Start the new datanode */
-       doImmediate(host, NULL, "pg_ctl start -w -Z restoremode -D %s -o -i", dir);
+       /* Start the new datanode, in binary upgrade mode */
+       doImmediate(host, NULL, "pg_ctl start -w -Z restoremode -D %s -o '-i -b'", dir);
 
        /* Allow the new datanode to start up by sleeping for a couple of seconds */
        pg_usleep(2000000L);

diff --git a/src/bin/pgxc_ctl/t/040_add_nodes.pl b/src/bin/pgxc_ctl/t/040_add_nodes.pl
new file mode 100755 (executable)
index 0000000..df19fb7
--- /dev/null
+++ b/src/bin/pgxc_ctl/t/040_add_nodes.pl
@@ -0,0 +1,74 @@
+use strict;
+use warnings;
+use Cwd;
+use Config;
+use TestLib;
+use Test::More tests => 9;
+
+my $dataDirRoot="~/DATA/pgxl/nodes/";
+$ENV{'PGXC_CTL_HOME'} = '/tmp/pgxc_ctl';
+my $PGXC_CTL_HOME=$ENV{'PGXC_CTL_HOME'};
+
+#delete related dirs for cleanup
+system("rm -rf $dataDirRoot");
+system("rm -rf $PGXC_CTL_HOME");
+
+my $GTM_HOST = "localhost";
+my $COORD1_HOST = "localhost";
+my $COORD2_HOST = "localhost";
+my $COORD3_HOST = "localhost";
+my $COORD1_PORT=30001;
+my $COORD2_PORT=30002;
+my $COORD3_PORT=30003;
+my $DN1_HOST = "localhost";
+my $DN2_HOST = "localhost";
+my $DN3_HOST = "localhost";
+my $TEST_DB = "testdb";
+my $DEFAULT_DB = "postgres";
+
+system_or_bail 'pgxc_ctl', 'prepare', 'config', 'empty' ;
+
+system_or_bail 'pgxc_ctl', 'add', 'gtm', 'master', 'gtm', "$GTM_HOST", '20001', "$dataDirRoot/gtm" ;
+
+system_or_bail 'pgxc_ctl', 'add', 'coordinator', 'master', 'coord1', "$COORD1_HOST", '30001', '30011', "$dataDirRoot/coord_master.1", 'none', 'none';
+
+system_or_bail 'pgxc_ctl', 'add', 'datanode', 'master', 'dn1', "$DN1_HOST", '40001', '40011', "$dataDirRoot/dn_master.1", 'none', 'none', 'none' ;
+
+system_or_bail 'pgxc_ctl', 'add', 'datanode', 'master', 'dn2', "$DN2_HOST", '40002', '40012', "$dataDirRoot/dn_master.2", 'none', 'none', 'none' ;
+
+system_or_bail 'pgxc_ctl', 'monitor', 'all' ;
+
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$DEFAULT_DB",'-c', "CREATE DATABASE testdb;";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "create table testtab (a text, b int, c text) distribute by hash(b);";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "alter table testtab drop column a;";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "alter table testtab add column d int;";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "alter table testtab add column a int;";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "insert into testtab values (1, 'foo', 10, 2);";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "insert into testtab values (2, 'foo', 10, 3);";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "insert into testtab values (3, 'foo', 10, 4);";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "insert into testtab values (4, 'foo', 10, 5);";
+system_or_bail 'psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "insert into testtab values (5, 'foo', 10, 1);";
+command_ok(['psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "select count(*) from testtab;"], 'select count coord1');
+command_ok(['psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "select sum(a) from testtab;"], 'select sum(a) coord1');
+command_ok(['psql', '-p', "$COORD1_PORT", "$TEST_DB",'-c', "select sum(b) from testtab;"], 'select sum(b) coord1');
+
+system_or_bail 'pgxc_ctl', 'add', 'coordinator', 'master', 'coord2', "$COORD2_HOST", '30002', '30012', "$dataDirRoot/coord_master.2", 'none', 'none';
+command_ok(['psql', '-p', "$COORD2_PORT", "$TEST_DB",'-c', "select count(*) from testtab;"], 'select count coord2');
+command_ok(['psql', '-p', "$COORD2_PORT", "$TEST_DB",'-c', "select sum(a) from testtab;"], 'select sum(a) coord2');
+command_ok(['psql', '-p', "$COORD2_PORT", "$TEST_DB",'-c', "select sum(b) from testtab;"], 'select sum(b) coord2');
+
+system_or_bail 'pgxc_ctl', 'add', 'datanode', 'master', 'dn3', "$DN3_HOST", '40003', '40013', "$dataDirRoot/dn_master.3", 'none', 'none', 'none' ;
+system_or_bail 'psql', '-p', "$COORD2_PORT", "$TEST_DB",'-c', "alter table testtab add node (dn3);";
+
+system_or_bail 'pgxc_ctl', 'add', 'coordinator', 'master', 'coord3', "$COORD3_HOST", '30003', '30013', "$dataDirRoot/coord_master.3", 'none', 'none' ;
+system_or_bail 'pgxc_ctl', 'monitor', 'all' ;
+command_ok(['psql', '-p', "$COORD3_PORT", "$TEST_DB",'-c', "select count(*) from testtab;"], 'select count coord3');
+command_ok(['psql', '-p', "$COORD3_PORT", "$TEST_DB",'-c', "select sum(a) from testtab;"], 'select sum(a) coord3');
+command_ok(['psql', '-p', "$COORD3_PORT", "$TEST_DB",'-c', "select sum(b) from testtab;"], 'select sum(b) coord3');
+
+#add cleanup
+system_or_bail 'pgxc_ctl', 'clean', 'all' ;
+
+#delete related dirs for cleanup
+system("rm -rf $dataDirRoot");
+system("rm -rf $PGXC_CTL_HOME");