diff --git a/contrib/pax_storage/src/test/isolation2/expected/commit_transaction_block_checkpoint.out b/contrib/pax_storage/src/test/isolation2/expected/commit_transaction_block_checkpoint.out index 0857b3b7a2a..ed6ec2f3566 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/commit_transaction_block_checkpoint.out +++ b/contrib/pax_storage/src/test/isolation2/expected/commit_transaction_block_checkpoint.out @@ -1,5 +1,8 @@ -- TEST 1: block checkpoint on segments +SET debug_disable_distributed_snapshot = on; +SET + -- pause the 2PC after setting inCommit flag select gp_inject_fault_infinite('before_xlog_xact_commit_prepared', 'suspend', 3); gp_inject_fault_infinite diff --git a/contrib/pax_storage/src/test/isolation2/expected/prepare_limit.out b/contrib/pax_storage/src/test/isolation2/expected/prepare_limit.out index d12b869db47..fb164e6ce0b 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/prepare_limit.out +++ b/contrib/pax_storage/src/test/isolation2/expected/prepare_limit.out @@ -32,6 +32,19 @@ CREATE Success: (1 row) +-- This test uses a suspended commit as a test barrier. Keep the concurrent +-- writers from waiting for that barrier while obtaining their snapshots. +1: SET debug_disable_distributed_snapshot = on; +SET +2: SET debug_disable_distributed_snapshot = on; +SET +3: SET debug_disable_distributed_snapshot = on; +SET +4: SET debug_disable_distributed_snapshot = on; +SET +5: SET debug_disable_distributed_snapshot = on; +SET + -- Note first insert after table create triggers auto_stats and leads to 2pc -- transaction. diff --git a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out index ed5e3d201b8..5bf16eb0230 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out +++ b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out @@ -9,6 +9,10 @@ select gp_inject_fault('wal_sender_loop', 'reset', 2); Success: (1 row) +-- This session coordinates a suspended commit rather than testing snapshots. +SET debug_disable_distributed_snapshot = on; +SET + create or replace function wait_for_replication(iterations int) returns bool as $$ begin /* in func */ for i in 1 .. iterations loop /* in func */ if exists (select wait_event from pg_stat_activity where sess_id in (select sess_id from store_session_id) and wait_event = 'SyncRep') then /* in func */ return true; /* in func */ end if; /* in func */ perform pg_sleep(0.1); /* in func */ perform pg_stat_clear_snapshot(); /* in func */ end loop; /* in func */ return false; /* in func */ end; /* in func */ $$ language plpgsql VOLATILE; CREATE diff --git a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out index bc706f16321..ef5e330d2ba 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out +++ b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out @@ -15,6 +15,9 @@ CREATE -- Suspend to hit commit-prepared point on segment (as we are -- interested in testing Commit here and not really Prepare) +-- The following waits are test coordination, not snapshot coverage. +SET debug_disable_distributed_snapshot = on; +SET select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where role='p' and content = 0; gp_inject_fault_infinite -------------------------- diff --git a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out index a410a91b754..18fa83d1366 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out +++ b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out @@ -8,6 +8,12 @@ 1: create table t_wait_lsn(a int); CREATE +-- These sessions coordinate suspended commits rather than test snapshots. +2: SET debug_disable_distributed_snapshot = on; +SET +3: SET debug_disable_distributed_snapshot = on; +SET + -- suspend segment 0 before performing 'COMMIT PREPARED' 2: select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; gp_inject_fault_infinite diff --git a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out index d27c4bbae4e..2268f7f994f 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out +++ b/contrib/pax_storage/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out @@ -182,6 +182,8 @@ INSERT 1 -- Scenario 3: Commit-Prepare received on primary but not acknowledged to master -- NOTICE: Don't use session 2 again because it's cached gang is invalid +4: SET debug_disable_distributed_snapshot = on; +SET 1:SELECT gp_inject_fault_infinite('finish_prepared_start_of_function', 'infinite_loop', dbid) FROM gp_segment_configuration WHERE content = 2 AND role = 'p'; gp_inject_fault_infinite -------------------------- diff --git a/contrib/pax_storage/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql b/contrib/pax_storage/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql index 5101b6132fe..cb4be72f8a9 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql @@ -1,5 +1,7 @@ -- TEST 1: block checkpoint on segments +SET debug_disable_distributed_snapshot = on; + -- pause the 2PC after setting inCommit flag select gp_inject_fault_infinite('before_xlog_xact_commit_prepared', 'suspend', 3); diff --git a/contrib/pax_storage/src/test/isolation2/sql/prepare_limit.sql b/contrib/pax_storage/src/test/isolation2/sql/prepare_limit.sql index 71aa13d02fe..1538574cb77 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/prepare_limit.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/prepare_limit.sql @@ -17,6 +17,14 @@ 5: select gp_inject_fault_infinite('dtm_before_insert_forget_comitted', 'suspend', 1); +-- This test uses a suspended commit as a test barrier. Keep the concurrent +-- writers from waiting for that barrier while obtaining their snapshots. +1: SET debug_disable_distributed_snapshot = on; +2: SET debug_disable_distributed_snapshot = on; +3: SET debug_disable_distributed_snapshot = on; +4: SET debug_disable_distributed_snapshot = on; +5: SET debug_disable_distributed_snapshot = on; + -- Note first insert after table create triggers auto_stats and leads to 2pc -- transaction. diff --git a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql index fef5e5ea811..06f94193d62 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql @@ -1,6 +1,9 @@ select gp_inject_fault('sync_rep_query_cancel', 'reset', 2); select gp_inject_fault('wal_sender_loop', 'reset', 2); +-- This session coordinates a suspended commit rather than testing snapshots. +SET debug_disable_distributed_snapshot = on; + create or replace function wait_for_replication(iterations int) returns bool as $$ begin /* in func */ diff --git a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql index 51c3a87d620..f2f4a0f8573 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql @@ -11,6 +11,8 @@ create table store_session_id(a int, sess_id int); -- Suspend to hit commit-prepared point on segment (as we are -- interested in testing Commit here and not really Prepare) +-- The following waits are test coordination, not snapshot coverage. +SET debug_disable_distributed_snapshot = on; select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where role='p' and content = 0; 1&: insert into die_commit_pending_replication values(2),(1); select gp_wait_until_triggered_fault('finish_prepared_start_of_function', 1, dbid) from gp_segment_configuration where role='p' and content = 0; diff --git a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql index e9e61f2d0d6..9e192433b09 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql @@ -7,6 +7,10 @@ 1: create table t_wait_lsn(a int); +-- These sessions coordinate suspended commits rather than test snapshots. +2: SET debug_disable_distributed_snapshot = on; +3: SET debug_disable_distributed_snapshot = on; + -- suspend segment 0 before performing 'COMMIT PREPARED' 2: select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; 1&: insert into t_wait_lsn values(2),(1); diff --git a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql index 9976b155da2..4a01361eca2 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql @@ -81,6 +81,7 @@ -- Scenario 3: Commit-Prepare received on primary but not acknowledged to master -- NOTICE: Don't use session 2 again because it's cached gang is invalid +4: SET debug_disable_distributed_snapshot = on; 1:SELECT gp_inject_fault_infinite('finish_prepared_start_of_function', 'infinite_loop', dbid) FROM gp_segment_configuration WHERE content = 2 AND role = 'p'; 1&:DROP TABLE tolerance_test_table; diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index 49b305f7cd0..53c8762b5b3 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1310,9 +1310,14 @@ def _get_phase2_status(self): try: dburl = dbconn.DbURL(dbname=self.dbname) - with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + # This catalog-only existence check must remain available while a + # distributed transaction is completing. The status table itself + # is distributed, so read it through the normal dispatcher below. + with closing(dbconn.connect(dburl, utility=True, encoding='UTF8')) as conn: if not dbconn.querySingleton(conn, status_table_exists_sql): return False + + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: status = dbconn.querySingleton(conn, sql) except Exception: # schema table not found diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand_status.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand_status.py index 1a5f286fbc6..ab1761a3660 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand_status.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand_status.py @@ -6,6 +6,8 @@ import os +from mock import call, patch + from gppylib.test.unit.gp_unittest import GpTestCase, run_tests from gppylib.commands import gp from gppylib.db import dbconn @@ -19,6 +21,23 @@ class Context(object): ctx = Context() +class GpExpandStatusConnectionMode(GpTestCase): + + @patch('gppylib.commands.gp.dbconn.connect') + @patch('gppylib.commands.gp.dbconn.DbURL') + @patch('gppylib.commands.gp.dbconn.querySingleton', side_effect=[1, 'SETUP DONE']) + def test_phase2_catalog_check_uses_utility_connection(self, query_singleton, dburl_cls, connect): + status = gp._GpExpandStatus() + status.dbname = 'testdb' + + self.assertTrue(status._get_phase2_status()) + self.assertEqual(status.status, 'SETUP DONE') + dburl_cls.assert_called_once_with(dbname='testdb') + self.assertEqual(connect.call_args_list, [ + call(dburl_cls.return_value, utility=True, encoding='UTF8'), + call(dburl_cls.return_value, encoding='UTF8'), + ]) + def get_gpexpand_status(): st = gp.get_gpexpand_status() st.dbname = ctx.dbname diff --git a/src/backend/cdb/cdbtm.c b/src/backend/cdb/cdbtm.c index 5733b21cadb..9d62e4cf987 100644 --- a/src/backend/cdb/cdbtm.c +++ b/src/backend/cdb/cdbtm.c @@ -129,6 +129,7 @@ static void retryAbortPrepared(void); static void doQEDistributedExplicitBegin(); static void currentDtxActivate(void); static void setCurrentDtxState(DtxState state); +static void markDtxCommitInProgress(void); static bool isDtxQueryDispatcher(void); static void performDtxProtocolCommitPrepared(const char *gid, bool raiseErrorIfNotFound); @@ -176,6 +177,21 @@ getDistributedTransactionId(void) return InvalidDistributedTransactionId; } +/* + * Mark the DTX before sending the first commit notification to a QE. The + * snapshot builder uses this state to wait for the QD's normal transaction + * cleanup, which keeps the proc-array and QE commit views atomic. + */ +static void +markDtxCommitInProgress(void) +{ + Assert(MyTmGxact->gxid != InvalidDistributedTransactionId); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + MyTmGxact->commitInProgress = true; + LWLockRelease(ProcArrayLock); +} + bool getDistributedTransactionIdentifier(char *id) { @@ -560,6 +576,8 @@ doNotifyingOnePhaseCommit(void) Assert(MyTmGxactLocal->state == DTX_STATE_ONE_PHASE_COMMIT); setCurrentDtxState(DTX_STATE_NOTIFYING_ONE_PHASE_COMMIT); + markDtxCommitInProgress(); + succeeded = currentDtxDispatchProtocolCommand(DTX_PROTOCOL_COMMAND_COMMIT_ONEPHASE, true); if (!succeeded) { @@ -587,8 +605,6 @@ doNotifyingCommitPrepared(void) Assert(MyTmGxactLocal->state == DTX_STATE_INSERTED_COMMITTED); setCurrentDtxState(DTX_STATE_NOTIFYING_COMMIT_PREPARED); - SIMPLE_FAULT_INJECTOR("dtm_broadcast_commit_prepared"); - /* * Acquire TwophaseCommitLock in shared mode to block any GPDB restore * points from being created while commit prepared messages are being @@ -596,6 +612,9 @@ doNotifyingCommitPrepared(void) */ LWLockAcquire(TwophaseCommitLock, LW_SHARED); + SIMPLE_FAULT_INJECTOR("dtm_broadcast_commit_prepared"); + markDtxCommitInProgress(); + savedInterruptHoldoffCount = InterruptHoldoffCount; Assert(MyTmGxactLocal->dtxSegments != NIL); @@ -1491,6 +1510,7 @@ resetTmGxact(void) Assert(MyTmGxact->gxid == InvalidDistributedTransactionId); MyTmGxact->xminDistributedSnapshot = InvalidDistributedTransactionId; MyTmGxact->includeInCkpt = false; + MyTmGxact->commitInProgress = false; MyTmGxact->sessionId = 0; MyTmGxactLocal->explicitBeginRemembered = false; diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 7a3078c5ca6..21bb3faab04 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -61,6 +61,7 @@ #include "port/atomics.h" #include "pgstat.h" #include "port/pg_lfind.h" +#include "storage/lmgr.h" #include "storage/proc.h" #include "storage/procarray.h" #include "storage/spin.h" @@ -75,6 +76,7 @@ #include "access/xact.h" /* setting the shared xid */ #include "cdb/cdbtm.h" #include "cdb/cdbvars.h" +#include "postmaster/fts.h" #include "utils/faultinjector.h" #include "utils/sharedsnapshot.h" #include "libpq/libpq-be.h" @@ -698,6 +700,7 @@ ProcArrayEndGxact(TMGXACT *tmGxact) pg_atomic_init_u64(&(tmGxact->atomic_gxid), InvalidDistributedTransactionId); tmGxact->xminDistributedSnapshot = InvalidDistributedTransactionId; tmGxact->includeInCkpt = false; + tmGxact->commitInProgress = false; tmGxact->sessionId = 0; /* @@ -2625,6 +2628,48 @@ DistributedSnapshotMappedEntry_Compare(const void *p1, const void *p2) return -1; } +/* + * Wait for DTX commit notifications that have started before taking a QD + * distributed snapshot. The caller must not hold ProcArrayLock. On return the + * lock is held in shared mode, so the caller can build its snapshot from the + * same proc-array state that it checked here. + */ +static void +WaitForDtxCommit(void) +{ + DistributedTransactionId waitGxid; + + for (;;) + { + waitGxid = InvalidDistributedTransactionId; + + LWLockAcquire(ProcArrayLock, LW_SHARED); + + for (int i = 0; i < procArray->numProcs; i++) + { + int pgprocno = procArray->pgprocnos[i]; + volatile TMGXACT *tmGxact = &allTmGxact[pgprocno]; + + DistributedTransactionId gxid = + pg_atomic_read_u64(&tmGxact->atomic_gxid); + + if (tmGxact == MyTmGxact || + gxid == InvalidDistributedTransactionId || + !tmGxact->commitInProgress) + continue; + + waitGxid = gxid; + break; + } + + if (waitGxid == InvalidDistributedTransactionId) + break; + + LWLockRelease(ProcArrayLock); + GxactLockTableWait(waitGxid); + } +} + /* * create distributed snapshot based on current visible distributed transaction */ @@ -3020,9 +3065,22 @@ GetSnapshotData(Snapshot snapshot, DtxContext distributedTransactionContext) /* * It is sufficient to get shared lock on ProcArrayLock, even if we are - * going to set MyProc->xmin. + * going to set MyProc->xmin. A DTX whose QE commit notification has + * started is different: wait until its QD cleanup completes before taking + * the snapshot, otherwise some QEs can expose the commit while others do + * not. + * + * FTS only reads coordinator configuration and probes segments through + * libpq. It must not wait for a user DTX here, because the probe can be + * needed to resolve that DTX. */ - LWLockAcquire(ProcArrayLock, LW_SHARED); + if (distributedTransactionContext == DTX_CONTEXT_QD_DISTRIBUTED_CAPABLE && + Gp_role != GP_ROLE_UTILITY && + !Debug_disable_distributed_snapshot && !am_ftsprobe && + needDistributedSnapshot) + WaitForDtxCommit(); + else + LWLockAcquire(ProcArrayLock, LW_SHARED); /* * GPDB_14_MERGE_FIXME: @@ -3283,9 +3341,14 @@ GetSnapshotData(Snapshot snapshot, DtxContext distributedTransactionContext) if (!TransactionIdIsValid(MyProc->xmin)) MyProc->xmin = TransactionXmin = xmin; - /* GP: QD takes a distributed snapshot iff QD not in retry phase and the query needs distributed snapshot */ - if (distributedTransactionContext == DTX_CONTEXT_QD_DISTRIBUTED_CAPABLE && !Debug_disable_distributed_snapshot - && needDistributedSnapshot) + /* + * GP: A QD takes a distributed snapshot iff it is not in retry phase and + * the query needs one. FTS only reads coordinator configuration while it + * probes segments, so it does not need a distributed snapshot. + */ + if (distributedTransactionContext == DTX_CONTEXT_QD_DISTRIBUTED_CAPABLE && + !Debug_disable_distributed_snapshot && !am_ftsprobe && + needDistributedSnapshot) { CreateDistributedSnapshot(ds); snapshot->haveDistribSnapshot = true; @@ -6671,4 +6734,4 @@ LoopBackendProc(BackendProcCallbackFunction func, void *args) (*func)(proc, args); } LWLockRelease(ProcArrayLock); -} \ No newline at end of file +} diff --git a/src/include/cdb/cdbtm.h b/src/include/cdb/cdbtm.h index dc05e1a79dd..116c1da9f7a 100644 --- a/src/include/cdb/cdbtm.h +++ b/src/include/cdb/cdbtm.h @@ -229,6 +229,8 @@ typedef struct TMGXACT DistributedTransactionId xminDistributedSnapshot; bool includeInCkpt; + /* True after QE commit notification starts, until the DTX ends. */ + bool commitInProgress; int sessionId; } TMGXACT; diff --git a/src/test/isolation2/Makefile b/src/test/isolation2/Makefile index 75a3c584b48..5058b5676c8 100644 --- a/src/test/isolation2/Makefile +++ b/src/test/isolation2/Makefile @@ -78,6 +78,9 @@ installcheck-expandshrink: install installcheck-isolation2: install $(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule +installcheck-distributed-snapshot: install + $(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 issue1465_distributed_snapshot + installcheck-isolation2-crash: install $(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_crash_schedule diff --git a/src/test/isolation2/expected/commit_transaction_block_checkpoint.out b/src/test/isolation2/expected/commit_transaction_block_checkpoint.out index 0857b3b7a2a..b04757a43d7 100644 --- a/src/test/isolation2/expected/commit_transaction_block_checkpoint.out +++ b/src/test/isolation2/expected/commit_transaction_block_checkpoint.out @@ -17,7 +17,9 @@ CREATE 2&: commit; -- wait for the fault to trigger since following checkpoint could be faster -select gp_wait_until_triggered_fault('before_xlog_xact_commit_prepared', 1, 3); +-1U: SET debug_disable_distributed_snapshot = on; +SET +-1U: select gp_wait_until_triggered_fault('before_xlog_xact_commit_prepared', 1, 3); gp_wait_until_triggered_fault ------------------------------- Success: @@ -27,7 +29,7 @@ select gp_wait_until_triggered_fault('before_xlog_xact_commit_prepared', 1, 3); 1U&: checkpoint; -- resume the 2PC after setting inCommit flag -select gp_inject_fault('before_xlog_xact_commit_prepared', 'reset', 3); +-1U: select gp_inject_fault('before_xlog_xact_commit_prepared', 'reset', 3); gp_inject_fault ----------------- Success: @@ -41,6 +43,8 @@ CHECKPOINT -- pause the CommitTransaction right before persistent table cleanup after -- notifyCommittedDtxTransaction() +1: SET debug_disable_distributed_snapshot = on; +SET select gp_inject_fault_infinite('onephase_transaction_commit', 'suspend', 1); gp_inject_fault_infinite -------------------------- @@ -57,7 +61,7 @@ DROP 2&: commit; -- wait for the fault to trigger since following checkpoint could be faster -select gp_wait_until_triggered_fault('onephase_transaction_commit', 1, 1); +-1U: select gp_wait_until_triggered_fault('onephase_transaction_commit', 1, 1); gp_wait_until_triggered_fault ------------------------------- Success: diff --git a/src/test/isolation2/expected/gdd/concurrent_update.out b/src/test/isolation2/expected/gdd/concurrent_update.out index 761e3f681a1..f9485985140 100644 --- a/src/test/isolation2/expected/gdd/concurrent_update.out +++ b/src/test/isolation2/expected/gdd/concurrent_update.out @@ -1,4 +1,7 @@ -- Test concurrent update a table with a varying length type +-1U:@db_name postgres: CREATE EXTENSION IF NOT EXISTS gp_inject_fault; +CREATE + CREATE TABLE t_concurrent_update(a int, b int, c char(84)); CREATE INSERT INTO t_concurrent_update VALUES(1,1,'test'); @@ -54,31 +57,35 @@ SET Success: (1 row) 2&: END; -1: select gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; +-- Use utility mode for fault coordination. A regular QD snapshot would wait +-- for the suspended DTX that this query is responsible for releasing. +-1U:@db_name postgres: select gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) +-- Release the suspended commit before waiting for its QD cleanup. +-1U:@db_name postgres: select gp_inject_fault('before_xact_end_procarray', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; + gp_inject_fault +----------------- + Success: +(1 row) -- transaction 3 should wait transaction 2 commit on master 3<: <... completed> UPDATE 1 3&: END; +-- A QD distributed snapshot waits for the commit cleanup to finish. +3<: <... completed> +END -- the query should not get the incorrect distributed snapshot: transaction 1 in-progress -- and transaction 2 finished 1: SELECT * FROM t_concurrent_update; - a | b ----+--- - 1 | 1 -(1 row) -1: select gp_inject_fault('before_xact_end_procarray', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; - gp_inject_fault ------------------ - Success: + a | b +---+---- + 1 | 21 (1 row) 2<: <... completed> END -3<: <... completed> -END 2q: ... 3q: ... @@ -111,14 +118,14 @@ SET (1 row) 5&: UPDATE t_concurrent_update SET b=b+10 WHERE a=1; -6: SELECT gp_wait_until_triggered_fault('before_get_distributed_xid', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_get_distributed_xid', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) 4: END; END -4: SELECT gp_inject_fault('before_get_distributed_xid', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; +-1U:@db_name postgres: SELECT gp_inject_fault('before_get_distributed_xid', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; gp_inject_fault ----------------- Success: diff --git a/src/test/isolation2/expected/gdd/concurrent_update_optimizer.out b/src/test/isolation2/expected/gdd/concurrent_update_optimizer.out index 4a04cd86a21..6b9a7cc50fe 100644 --- a/src/test/isolation2/expected/gdd/concurrent_update_optimizer.out +++ b/src/test/isolation2/expected/gdd/concurrent_update_optimizer.out @@ -1,4 +1,7 @@ -- Test concurrent update a table with a varying length type +-1U:@db_name postgres: CREATE EXTENSION IF NOT EXISTS gp_inject_fault; +CREATE + CREATE TABLE t_concurrent_update(a int, b int, c char(84)); CREATE INSERT INTO t_concurrent_update VALUES(1,1,'test'); @@ -54,31 +57,35 @@ SET Success: (1 row) 2&: END; -1: select gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; +-- Use utility mode for fault coordination. A regular QD snapshot would wait +-- for the suspended DTX that this query is responsible for releasing. +-1U:@db_name postgres: select gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) +-- Release the suspended commit before waiting for its QD cleanup. +-1U:@db_name postgres: select gp_inject_fault('before_xact_end_procarray', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; + gp_inject_fault +----------------- + Success: +(1 row) -- transaction 3 should wait transaction 2 commit on master 3<: <... completed> UPDATE 1 3&: END; +-- A QD distributed snapshot waits for the commit cleanup to finish. +3<: <... completed> +END -- the query should not get the incorrect distributed snapshot: transaction 1 in-progress -- and transaction 2 finished 1: SELECT * FROM t_concurrent_update; - a | b ----+--- - 1 | 1 -(1 row) -1: select gp_inject_fault('before_xact_end_procarray', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; - gp_inject_fault ------------------ - Success: + a | b +---+---- + 1 | 21 (1 row) 2<: <... completed> END -3<: <... completed> -END 2q: ... 3q: ... @@ -111,14 +118,14 @@ SET (1 row) 5&: UPDATE t_concurrent_update SET b=b+10 WHERE a=1; -6: SELECT gp_wait_until_triggered_fault('before_get_distributed_xid', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_get_distributed_xid', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) 4: END; END -4: SELECT gp_inject_fault('before_get_distributed_xid', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; +-1U:@db_name postgres: SELECT gp_inject_fault('before_get_distributed_xid', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; gp_inject_fault ----------------- Success: diff --git a/src/test/isolation2/expected/hot_standby/faults.out b/src/test/isolation2/expected/hot_standby/faults.out index 2eb16b37229..845d5cb6f61 100644 --- a/src/test/isolation2/expected/hot_standby/faults.out +++ b/src/test/isolation2/expected/hot_standby/faults.out @@ -221,6 +221,10 @@ select wait_until_all_segments_synchronized(); 1: create table tt_hs_dtx(a int); CREATE +-- Session 2 coordinates a suspended commit rather than testing snapshots. +2: SET debug_disable_distributed_snapshot = on; +SET + -- inject fault to repeatedly fail the COMMIT PREPARE phase of 2PC, which ensures that the dtx cannot finish even by the dtx recovery process. select gp_inject_fault_infinite('finish_commit_prepared', 'error', dbid) from gp_segment_configuration where content=1 and role='p'; gp_inject_fault_infinite @@ -324,4 +328,3 @@ server closed the connection unexpectedly ----------------- Success: (1 row) - diff --git a/src/test/isolation2/expected/hot_standby/transaction_isolation.out b/src/test/isolation2/expected/hot_standby/transaction_isolation.out index 3990bd7cd56..a7455fc46a6 100644 --- a/src/test/isolation2/expected/hot_standby/transaction_isolation.out +++ b/src/test/isolation2/expected/hot_standby/transaction_isolation.out @@ -96,6 +96,9 @@ CREATE (0 rows) -- reset +-- This control session must release a commit already suspended on a QE. +3: SET debug_disable_distributed_snapshot = on; +SET 3: select gp_inject_fault('qe_start_prepared', 'reset',dbid) from gp_segment_configuration where content=0 and role='p'; gp_inject_fault ----------------- @@ -315,6 +318,9 @@ CREATE 1&: insert into hs_t5 select i, 'in-progress' from generate_series(1,10) i; -- now run some dtx and completed, and primary conducts a checkpoint +-- This independent DTX must not wait for the suspended commit above. +2: SET debug_disable_distributed_snapshot = on; +SET 2: insert into hs_t5 values(1, 'commited'); INSERT 1 2: insert into hs_t6 select i, 'committed' from generate_series(1,10) i; @@ -369,6 +375,8 @@ server closed the connection unexpectedly (1 row) 1<: <... completed> INSERT 10 +2: RESET debug_disable_distributed_snapshot; +RESET -- standby should see all rows now -1S: select * from hs_t5; @@ -463,6 +471,8 @@ INSERT 10 DELETE 11 2<: <... completed> DELETE 10 +3: RESET debug_disable_distributed_snapshot; +RESET -- standby now see those deletes -1S: select * from hs_t5; diff --git a/src/test/isolation2/expected/issue1465_distributed_snapshot.out b/src/test/isolation2/expected/issue1465_distributed_snapshot.out new file mode 100644 index 00000000000..8b734f90263 --- /dev/null +++ b/src/test/isolation2/expected/issue1465_distributed_snapshot.out @@ -0,0 +1,122 @@ +-- Verify that a distributed snapshot cannot retain a gxid after the QE has +-- committed it. Session 1 is opened before the fault is installed because +-- the fault is scoped to the isolation2test database. +-- The control connection uses postgres, so install the debug extension there +-- as well. --load-extension only installs it in isolation2test. +-1U:@db_name postgres: CREATE EXTENSION IF NOT EXISTS gp_inject_fault; +CREATE + +CREATE TABLE issue1465_snapshot (k int, v int) DISTRIBUTED REPLICATED; +CREATE +INSERT INTO issue1465_snapshot VALUES (1, 400); +INSERT 1 + +1: SELECT 1; + ?column? +---------- + 1 +(1 row) +1: BEGIN; +BEGIN +1: UPDATE issue1465_snapshot SET v = 500 WHERE k = 1; +UPDATE 1 +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'suspend', '', 'isolation2test', '', 1, 1, 0, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +1&: COMMIT; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) + +2: BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN +2&: SELECT k, v FROM issue1465_snapshot ORDER BY k; + +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'resume', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +1<: <... completed> +COMMIT +2<: <... completed> + k | v +---+----- + 1 | 500 +(1 row) +-1U:@db_name postgres: SELECT gp_inject_fault('all', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_inject_fault +----------------- + Success: +(1 row) + +-- The same snapshot must retain the committed version after the writer exits. +2: SELECT k, v FROM issue1465_snapshot ORDER BY k; + k | v +---+----- + 1 | 500 +(1 row) +2: ROLLBACK; +ROLLBACK + +DROP TABLE issue1465_snapshot; +DROP + +-- The one-phase path also notifies a QE before QD transaction cleanup. +CREATE TABLE issue1465_onephase (k int, v int) DISTRIBUTED BY (k); +CREATE +INSERT INTO issue1465_onephase VALUES (1, 400); +INSERT 1 + +1: BEGIN; +BEGIN +1: UPDATE issue1465_onephase SET v = 500 WHERE k = 1; +UPDATE 1 +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'suspend', '', 'isolation2test', '', 1, 1, 0, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +1&: COMMIT; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) + +2: BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN +2&: SELECT k, v FROM issue1465_onephase WHERE k = 1; + +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'resume', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +1<: <... completed> +COMMIT +2<: <... completed> + k | v +---+----- + 1 | 500 +(1 row) +-1U:@db_name postgres: SELECT gp_inject_fault('all', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + gp_inject_fault +----------------- + Success: +(1 row) + +2: SELECT k, v FROM issue1465_onephase WHERE k = 1; + k | v +---+----- + 1 | 500 +(1 row) +2: ROLLBACK; +ROLLBACK + +DROP TABLE issue1465_onephase; +DROP diff --git a/src/test/isolation2/expected/prepare_limit.out b/src/test/isolation2/expected/prepare_limit.out index d12b869db47..fb164e6ce0b 100644 --- a/src/test/isolation2/expected/prepare_limit.out +++ b/src/test/isolation2/expected/prepare_limit.out @@ -32,6 +32,19 @@ CREATE Success: (1 row) +-- This test uses a suspended commit as a test barrier. Keep the concurrent +-- writers from waiting for that barrier while obtaining their snapshots. +1: SET debug_disable_distributed_snapshot = on; +SET +2: SET debug_disable_distributed_snapshot = on; +SET +3: SET debug_disable_distributed_snapshot = on; +SET +4: SET debug_disable_distributed_snapshot = on; +SET +5: SET debug_disable_distributed_snapshot = on; +SET + -- Note first insert after table create triggers auto_stats and leads to 2pc -- transaction. diff --git a/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out b/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out index ed5e3d201b8..5bf16eb0230 100644 --- a/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out +++ b/src/test/isolation2/expected/segwalrep/cancel_commit_pending_replication.out @@ -9,6 +9,10 @@ select gp_inject_fault('wal_sender_loop', 'reset', 2); Success: (1 row) +-- This session coordinates a suspended commit rather than testing snapshots. +SET debug_disable_distributed_snapshot = on; +SET + create or replace function wait_for_replication(iterations int) returns bool as $$ begin /* in func */ for i in 1 .. iterations loop /* in func */ if exists (select wait_event from pg_stat_activity where sess_id in (select sess_id from store_session_id) and wait_event = 'SyncRep') then /* in func */ return true; /* in func */ end if; /* in func */ perform pg_sleep(0.1); /* in func */ perform pg_stat_clear_snapshot(); /* in func */ end loop; /* in func */ return false; /* in func */ end; /* in func */ $$ language plpgsql VOLATILE; CREATE diff --git a/src/test/isolation2/expected/segwalrep/commit_blocking.out b/src/test/isolation2/expected/segwalrep/commit_blocking.out index 5edac6d59f5..07ad665059e 100644 --- a/src/test/isolation2/expected/segwalrep/commit_blocking.out +++ b/src/test/isolation2/expected/segwalrep/commit_blocking.out @@ -90,6 +90,10 @@ INSERT 1 3&: commit; -- this should not block due to direct dispatch to primary with active synced mirror +-- Do not wait for the blocked commit: this test exercises direct dispatch on +-- another primary while synchronous replication is waiting. +4: SET debug_disable_distributed_snapshot = on; +SET 4: insert into segwalrep_commit_blocking values (6); INSERT 1 diff --git a/src/test/isolation2/expected/segwalrep/commit_blocking_on_standby.out b/src/test/isolation2/expected/segwalrep/commit_blocking_on_standby.out index c5eb5f674b8..7dfe1d276e6 100644 --- a/src/test/isolation2/expected/segwalrep/commit_blocking_on_standby.out +++ b/src/test/isolation2/expected/segwalrep/commit_blocking_on_standby.out @@ -22,6 +22,10 @@ select gp_inject_fault_infinite('walrecv_skip_flush', 'skip', dbid) from gp_segm Success: (1 row) +-- This session monitors and releases deliberately blocked synchronous commits. +SET debug_disable_distributed_snapshot = on; +SET + begin; BEGIN create or replace function wait_for_pg_stat_activity(timeout_secs int) returns void as $$ declare c int; /* in func */ i int; /* in func */ begin c := 0; /* in func */ i := 0; /* in func */ while c < 1 and i < timeout_secs*2 loop select count(*) into c from pg_stat_activity where wait_event = 'SyncRep'; /* in func */ perform pg_sleep(0.5); /* in func */ perform pg_stat_clear_snapshot(); /* in func */ i := i + 1; /* in func */ end loop; /* in func */ if c < 1 then raise exception 'timeout waiting for command to get blocked'; /* in func */ end if; /* in func */ end; /* in func */ $$ language plpgsql; diff --git a/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out b/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out index bc706f16321..ef5e330d2ba 100644 --- a/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out +++ b/src/test/isolation2/expected/segwalrep/die_commit_pending_replication.out @@ -15,6 +15,9 @@ CREATE -- Suspend to hit commit-prepared point on segment (as we are -- interested in testing Commit here and not really Prepare) +-- The following waits are test coordination, not snapshot coverage. +SET debug_disable_distributed_snapshot = on; +SET select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where role='p' and content = 0; gp_inject_fault_infinite -------------------------- diff --git a/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out b/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out index ce415fc5f96..dfe9553d535 100644 --- a/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out +++ b/src/test/isolation2/expected/segwalrep/dtx_recovery_wait_lsn.out @@ -10,6 +10,16 @@ CREATE 5: create table t_wait_lsn2(a int); CREATE +-- These sessions coordinate suspended commits rather than test snapshots. +1: SET debug_disable_distributed_snapshot = on; +SET +2: SET debug_disable_distributed_snapshot = on; +SET +3: SET debug_disable_distributed_snapshot = on; +SET +5: SET debug_disable_distributed_snapshot = on; +SET + -- suspend segment 0 before performing 'COMMIT PREPARED' 2: select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; gp_inject_fault_infinite diff --git a/src/test/isolation2/expected/segwalrep/fts_unblock_primary.out b/src/test/isolation2/expected/segwalrep/fts_unblock_primary.out index 5f1afd5134a..4ea03a75247 100644 --- a/src/test/isolation2/expected/segwalrep/fts_unblock_primary.out +++ b/src/test/isolation2/expected/segwalrep/fts_unblock_primary.out @@ -78,6 +78,10 @@ INSERT 1 2&: commit; -- this should not block due to direct dispatch to primary with active synced mirror +-- Do not wait for the blocked commit: this test exercises direct dispatch on +-- another primary while synchronous replication is waiting. +SET debug_disable_distributed_snapshot = on; +SET insert into fts_unblock_primary values (2); INSERT 1 diff --git a/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out b/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out index d27c4bbae4e..2268f7f994f 100644 --- a/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out +++ b/src/test/isolation2/expected/segwalrep/twophase_tolerance_with_mirror_promotion.out @@ -182,6 +182,8 @@ INSERT 1 -- Scenario 3: Commit-Prepare received on primary but not acknowledged to master -- NOTICE: Don't use session 2 again because it's cached gang is invalid +4: SET debug_disable_distributed_snapshot = on; +SET 1:SELECT gp_inject_fault_infinite('finish_prepared_start_of_function', 'infinite_loop', dbid) FROM gp_segment_configuration WHERE content = 2 AND role = 'p'; gp_inject_fault_infinite -------------------------- diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 82d23731fb6..f3989e0dde4 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -76,6 +76,7 @@ test: instr_in_shmem_terminate test: vacuum_recently_dead_tuple_due_to_distributed_snapshot test: vacuum_full_interrupt test: distributedlog-bug +test: issue1465_distributed_snapshot test: invalidated_toast_index test: distributed_snapshot test: gp_collation diff --git a/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql b/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql index 5101b6132fe..f400724d9b6 100644 --- a/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql +++ b/src/test/isolation2/sql/commit_transaction_block_checkpoint.sql @@ -10,13 +10,14 @@ select gp_inject_fault_infinite('before_xlog_xact_commit_prepared', 'suspend', 3 2&: commit; -- wait for the fault to trigger since following checkpoint could be faster -select gp_wait_until_triggered_fault('before_xlog_xact_commit_prepared', 1, 3); +-1U: SET debug_disable_distributed_snapshot = on; +-1U: select gp_wait_until_triggered_fault('before_xlog_xact_commit_prepared', 1, 3); -- do checkpoint on segment content 1 in utility mode, and it should block 1U&: checkpoint; -- resume the 2PC after setting inCommit flag -select gp_inject_fault('before_xlog_xact_commit_prepared', 'reset', 3); +-1U: select gp_inject_fault('before_xlog_xact_commit_prepared', 'reset', 3); 2<: 1U<: @@ -24,6 +25,7 @@ select gp_inject_fault('before_xlog_xact_commit_prepared', 'reset', 3); -- pause the CommitTransaction right before persistent table cleanup after -- notifyCommittedDtxTransaction() +1: SET debug_disable_distributed_snapshot = on; select gp_inject_fault_infinite('onephase_transaction_commit', 'suspend', 1); -- trigger a 2PC, and it will block at commit; @@ -33,7 +35,7 @@ select gp_inject_fault_infinite('onephase_transaction_commit', 'suspend', 1); 2&: commit; -- wait for the fault to trigger since following checkpoint could be faster -select gp_wait_until_triggered_fault('onephase_transaction_commit', 1, 1); +-1U: select gp_wait_until_triggered_fault('onephase_transaction_commit', 1, 1); -- do checkpoint on master in utility mode, and it should block -1U&: checkpoint; diff --git a/src/test/isolation2/sql/gdd/concurrent_update.sql b/src/test/isolation2/sql/gdd/concurrent_update.sql index 44cf5916b1b..b4556140a9a 100644 --- a/src/test/isolation2/sql/gdd/concurrent_update.sql +++ b/src/test/isolation2/sql/gdd/concurrent_update.sql @@ -1,4 +1,6 @@ -- Test concurrent update a table with a varying length type +-1U:@db_name postgres: CREATE EXTENSION IF NOT EXISTS gp_inject_fault; + CREATE TABLE t_concurrent_update(a int, b int, c char(84)); INSERT INTO t_concurrent_update VALUES(1,1,'test'); @@ -30,16 +32,20 @@ DROP TABLE t_concurrent_update; -- transaction 2 suspend before commit, but it will wake up transaction 3 on segment 2: select gp_inject_fault('before_xact_end_procarray', 'suspend', '', 'isolation2test', '', 1, 1, 0, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; 2&: END; -1: select gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; +-- Use utility mode for fault coordination. A regular QD snapshot would wait +-- for the suspended DTX that this query is responsible for releasing. +-1U:@db_name postgres: select gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; +-- Release the suspended commit before waiting for its QD cleanup. +-1U:@db_name postgres: select gp_inject_fault('before_xact_end_procarray', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; -- transaction 3 should wait transaction 2 commit on master 3<: 3&: END; +-- A QD distributed snapshot waits for the commit cleanup to finish. +3<: -- the query should not get the incorrect distributed snapshot: transaction 1 in-progress -- and transaction 2 finished 1: SELECT * FROM t_concurrent_update; -1: select gp_inject_fault('before_xact_end_procarray', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1; 2<: -3<: 2q: 3q: @@ -59,9 +65,9 @@ DROP TABLE t_concurrent_update; 5: SELECT gp_inject_fault('before_get_distributed_xid', 'suspend', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; 5&: UPDATE t_concurrent_update SET b=b+10 WHERE a=1; -6: SELECT gp_wait_until_triggered_fault('before_get_distributed_xid', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_get_distributed_xid', 1, dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; 4: END; -4: SELECT gp_inject_fault('before_get_distributed_xid', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; +-1U:@db_name postgres: SELECT gp_inject_fault('before_get_distributed_xid', 'reset', dbid) FROM gp_segment_configuration WHERE role='p' AND content=1; 5<: 5: END; diff --git a/src/test/isolation2/sql/hot_standby/faults.sql b/src/test/isolation2/sql/hot_standby/faults.sql index b1be240916a..96f161d4724 100644 --- a/src/test/isolation2/sql/hot_standby/faults.sql +++ b/src/test/isolation2/sql/hot_standby/faults.sql @@ -92,6 +92,9 @@ select wait_until_all_segments_synchronized(); 1: create table tt_hs_dtx(a int); +-- Session 2 coordinates a suspended commit rather than testing snapshots. +2: SET debug_disable_distributed_snapshot = on; + -- inject fault to repeatedly fail the COMMIT PREPARE phase of 2PC, which ensures that the dtx cannot finish even by the dtx recovery process. select gp_inject_fault_infinite('finish_commit_prepared', 'error', dbid) from gp_segment_configuration where content=1 and role='p'; @@ -123,4 +126,3 @@ select gp_inject_fault_infinite('finish_commit_prepared', 'error', dbid) from gp -1S: select wait_until_all_segments_synchronized(); 1: select gp_inject_fault('before_read_command', 'reset', dbid) from gp_segment_configuration where content=-1 and role='p'; 1: select gp_inject_fault('fts_probe', 'reset', dbid) from gp_segment_configuration where role='p' and content=-1; - diff --git a/src/test/isolation2/sql/hot_standby/transaction_isolation.sql b/src/test/isolation2/sql/hot_standby/transaction_isolation.sql index 68945228313..f43667833ad 100644 --- a/src/test/isolation2/sql/hot_standby/transaction_isolation.sql +++ b/src/test/isolation2/sql/hot_standby/transaction_isolation.sql @@ -36,6 +36,8 @@ -1S: select * from hs_dtx2; -- reset +-- This control session must release a commit already suspended on a QE. +3: SET debug_disable_distributed_snapshot = on; 3: select gp_inject_fault('qe_start_prepared', 'reset',dbid) from gp_segment_configuration where content=0 and role='p'; 3: select gp_inject_fault('qe_start_commit_prepared', 'reset',dbid) from gp_segment_configuration where content=1 and role='p'; 1<: @@ -111,6 +113,8 @@ 1&: insert into hs_t5 select i, 'in-progress' from generate_series(1,10) i; -- now run some dtx and completed, and primary conducts a checkpoint +-- This independent DTX must not wait for the suspended commit above. +2: SET debug_disable_distributed_snapshot = on; 2: insert into hs_t5 values(1, 'commited'); 2: insert into hs_t6 select i, 'committed' from generate_series(1,10) i; 2: begin; @@ -129,6 +133,7 @@ 2: select gp_inject_fault('qe_start_commit_prepared', 'reset',dbid) from gp_segment_configuration where content=0 and role='p'; 1<: +2: RESET debug_disable_distributed_snapshot; -- standby should see all rows now -1S: select * from hs_t5; @@ -151,6 +156,7 @@ 1<: 2<: +3: RESET debug_disable_distributed_snapshot; -- standby now see those deletes -1S: select * from hs_t5; diff --git a/src/test/isolation2/sql/issue1465_distributed_snapshot.sql b/src/test/isolation2/sql/issue1465_distributed_snapshot.sql new file mode 100644 index 00000000000..ba4ab13edc7 --- /dev/null +++ b/src/test/isolation2/sql/issue1465_distributed_snapshot.sql @@ -0,0 +1,53 @@ +-- Verify that a distributed snapshot cannot retain a gxid after the QE has +-- committed it. Session 1 is opened before the fault is installed because +-- the fault is scoped to the isolation2test database. +-- The control connection uses postgres, so install the debug extension there +-- as well. --load-extension only installs it in isolation2test. +-1U:@db_name postgres: CREATE EXTENSION IF NOT EXISTS gp_inject_fault; + +CREATE TABLE issue1465_snapshot (k int, v int) DISTRIBUTED REPLICATED; +INSERT INTO issue1465_snapshot VALUES (1, 400); + +1: SELECT 1; +1: BEGIN; +1: UPDATE issue1465_snapshot SET v = 500 WHERE k = 1; +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'suspend', '', 'isolation2test', '', 1, 1, 0, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; +1&: COMMIT; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + +2: BEGIN ISOLATION LEVEL REPEATABLE READ; +2&: SELECT k, v FROM issue1465_snapshot ORDER BY k; + +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'resume', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; +1<: +2<: +-1U:@db_name postgres: SELECT gp_inject_fault('all', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + +-- The same snapshot must retain the committed version after the writer exits. +2: SELECT k, v FROM issue1465_snapshot ORDER BY k; +2: ROLLBACK; + +DROP TABLE issue1465_snapshot; + +-- The one-phase path also notifies a QE before QD transaction cleanup. +CREATE TABLE issue1465_onephase (k int, v int) DISTRIBUTED BY (k); +INSERT INTO issue1465_onephase VALUES (1, 400); + +1: BEGIN; +1: UPDATE issue1465_onephase SET v = 500 WHERE k = 1; +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'suspend', '', 'isolation2test', '', 1, 1, 0, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; +1&: COMMIT; +-1U:@db_name postgres: SELECT gp_wait_until_triggered_fault('before_xact_end_procarray', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + +2: BEGIN ISOLATION LEVEL REPEATABLE READ; +2&: SELECT k, v FROM issue1465_onephase WHERE k = 1; + +-1U:@db_name postgres: SELECT gp_inject_fault('before_xact_end_procarray', 'resume', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; +1<: +2<: +-1U:@db_name postgres: SELECT gp_inject_fault('all', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = -1; + +2: SELECT k, v FROM issue1465_onephase WHERE k = 1; +2: ROLLBACK; + +DROP TABLE issue1465_onephase; diff --git a/src/test/isolation2/sql/prepare_limit.sql b/src/test/isolation2/sql/prepare_limit.sql index 71aa13d02fe..1538574cb77 100644 --- a/src/test/isolation2/sql/prepare_limit.sql +++ b/src/test/isolation2/sql/prepare_limit.sql @@ -17,6 +17,14 @@ 5: select gp_inject_fault_infinite('dtm_before_insert_forget_comitted', 'suspend', 1); +-- This test uses a suspended commit as a test barrier. Keep the concurrent +-- writers from waiting for that barrier while obtaining their snapshots. +1: SET debug_disable_distributed_snapshot = on; +2: SET debug_disable_distributed_snapshot = on; +3: SET debug_disable_distributed_snapshot = on; +4: SET debug_disable_distributed_snapshot = on; +5: SET debug_disable_distributed_snapshot = on; + -- Note first insert after table create triggers auto_stats and leads to 2pc -- transaction. diff --git a/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql b/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql index fef5e5ea811..06f94193d62 100644 --- a/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql +++ b/src/test/isolation2/sql/segwalrep/cancel_commit_pending_replication.sql @@ -1,6 +1,9 @@ select gp_inject_fault('sync_rep_query_cancel', 'reset', 2); select gp_inject_fault('wal_sender_loop', 'reset', 2); +-- This session coordinates a suspended commit rather than testing snapshots. +SET debug_disable_distributed_snapshot = on; + create or replace function wait_for_replication(iterations int) returns bool as $$ begin /* in func */ diff --git a/src/test/isolation2/sql/segwalrep/commit_blocking.sql b/src/test/isolation2/sql/segwalrep/commit_blocking.sql index aeac3756b3a..fb8f630d2a2 100644 --- a/src/test/isolation2/sql/segwalrep/commit_blocking.sql +++ b/src/test/isolation2/sql/segwalrep/commit_blocking.sql @@ -40,6 +40,9 @@ select gp_request_fts_probe_scan(); 3&: commit; -- this should not block due to direct dispatch to primary with active synced mirror +-- Do not wait for the blocked commit: this test exercises direct dispatch on +-- another primary while synchronous replication is waiting. +4: SET debug_disable_distributed_snapshot = on; 4: insert into segwalrep_commit_blocking values (6); -- bring the mirror back up diff --git a/src/test/isolation2/sql/segwalrep/commit_blocking_on_standby.sql b/src/test/isolation2/sql/segwalrep/commit_blocking_on_standby.sql index b45691aab94..2f7b2cd529f 100644 --- a/src/test/isolation2/sql/segwalrep/commit_blocking_on_standby.sql +++ b/src/test/isolation2/sql/segwalrep/commit_blocking_on_standby.sql @@ -15,6 +15,9 @@ select application_name, state from pg_stat_replication; select gp_inject_fault_infinite('walrecv_skip_flush', 'skip', dbid) from gp_segment_configuration where content=-1 and role='m'; +-- This session monitors and releases deliberately blocked synchronous commits. +SET debug_disable_distributed_snapshot = on; + begin; create or replace function wait_for_pg_stat_activity(timeout_secs int) returns void as $$ diff --git a/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql b/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql index 51c3a87d620..f2f4a0f8573 100644 --- a/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql +++ b/src/test/isolation2/sql/segwalrep/die_commit_pending_replication.sql @@ -11,6 +11,8 @@ create table store_session_id(a int, sess_id int); -- Suspend to hit commit-prepared point on segment (as we are -- interested in testing Commit here and not really Prepare) +-- The following waits are test coordination, not snapshot coverage. +SET debug_disable_distributed_snapshot = on; select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where role='p' and content = 0; 1&: insert into die_commit_pending_replication values(2),(1); select gp_wait_until_triggered_fault('finish_prepared_start_of_function', 1, dbid) from gp_segment_configuration where role='p' and content = 0; diff --git a/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql b/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql index 8cd7666e236..331af3fa09a 100644 --- a/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql +++ b/src/test/isolation2/sql/segwalrep/dtx_recovery_wait_lsn.sql @@ -8,6 +8,12 @@ 1: create table t_wait_lsn(a int); 5: create table t_wait_lsn2(a int); +-- These sessions coordinate suspended commits rather than test snapshots. +1: SET debug_disable_distributed_snapshot = on; +2: SET debug_disable_distributed_snapshot = on; +3: SET debug_disable_distributed_snapshot = on; +5: SET debug_disable_distributed_snapshot = on; + -- suspend segment 0 before performing 'COMMIT PREPARED' 2: select gp_inject_fault_infinite('finish_prepared_start_of_function', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; 1&: insert into t_wait_lsn values(2),(1); diff --git a/src/test/isolation2/sql/segwalrep/fts_unblock_primary.sql b/src/test/isolation2/sql/segwalrep/fts_unblock_primary.sql index 00551855a31..bab1fcb67d2 100644 --- a/src/test/isolation2/sql/segwalrep/fts_unblock_primary.sql +++ b/src/test/isolation2/sql/segwalrep/fts_unblock_primary.sql @@ -33,6 +33,9 @@ select gp_request_fts_probe_scan(); 2&: commit; -- this should not block due to direct dispatch to primary with active synced mirror +-- Do not wait for the blocked commit: this test exercises direct dispatch on +-- another primary while synchronous replication is waiting. +SET debug_disable_distributed_snapshot = on; insert into fts_unblock_primary values (2); -- resume FTS probes diff --git a/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql b/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql index 9976b155da2..4a01361eca2 100644 --- a/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql +++ b/src/test/isolation2/sql/segwalrep/twophase_tolerance_with_mirror_promotion.sql @@ -81,6 +81,7 @@ -- Scenario 3: Commit-Prepare received on primary but not acknowledged to master -- NOTICE: Don't use session 2 again because it's cached gang is invalid +4: SET debug_disable_distributed_snapshot = on; 1:SELECT gp_inject_fault_infinite('finish_prepared_start_of_function', 'infinite_loop', dbid) FROM gp_segment_configuration WHERE content = 2 AND role = 'p'; 1&:DROP TABLE tolerance_test_table;