From: Heikki Linnakangas Date: Mon, 12 May 2014 11:29:58 +0000 (+0300) Subject: Merge remote-tracking branch 'origin/master' into csn X-Git-Url: http://git.postgresql.org/gitweb/static/gitweb.js?a=commitdiff_plain;h=b80b3adb6dc7c48a52abb30ba3349929175f778e;p=users%2Fheikki%2Fpostgres.git Merge remote-tracking branch 'origin/master' into csn Conflicts: src/backend/access/heap/heapam.c src/backend/access/transam/clog.c src/backend/access/transam/transam.c src/backend/access/transam/xact.c src/backend/commands/cluster.c src/backend/commands/indexcmds.c src/backend/replication/logical/decode.c src/backend/replication/logical/reorderbuffer.c src/backend/replication/logical/snapbuild.c src/backend/replication/walsender.c src/backend/storage/ipc/procarray.c src/backend/utils/time/tqual.c src/include/pgstat.h src/include/replication/snapbuild.h src/include/storage/proc.h src/include/utils/snapshot.h --- b80b3adb6dc7c48a52abb30ba3349929175f778e diff --cc src/backend/access/heap/heapam.c index d83cf1f3b1,405117a526..f877127deb --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@@ -5013,8 -5012,7 +5012,8 @@@ static HTSU_Resul test_lockmode_for_conflict(MultiXactStatus status, TransactionId xid, LockTupleMode mode, bool *needwait) { - MultiXactStatus wantedstatus; + MultiXactStatus wantedstatus; + TransactionIdStatus xidstatus; *needwait = false; wantedstatus = get_mxact_status_for_lock(mode, false); @@@ -5032,15 -5030,13 +5031,15 @@@ */ return HeapTupleSelfUpdated; } - else if (TransactionIdIsInProgress(xid)) + xidstatus = TransactionIdGetStatus(xid); + + if (xidstatus == XID_INPROGRESS) { /* - * If the locking transaction is running, what we do depends on whether - * the lock modes conflict: if they do, then we must wait for it to - * finish; otherwise we can fall through to lock this tuple version - * without waiting. + * If the locking transaction is running, what we do depends on + * whether the lock modes conflict: if they do, then we must wait for + * it to finish; otherwise we can fall through to lock this tuple + * version without waiting. */ if (DoLockModesConflict(LOCKMODE_from_mxstatus(status), LOCKMODE_from_mxstatus(wantedstatus))) @@@ -5609,8 -5607,7 +5608,8 @@@ FreezeMultiXactId(MultiXactId multi, ui */ if (ISUPDATE_from_mxstatus(members[i].status)) { - TransactionId xid = members[i].xid; + TransactionId xid = members[i].xid; + TransactionIdStatus xidstatus; /* * It's an update; should we keep it? If the transaction is known diff --cc src/backend/access/transam/xact.c index c16c297909,3e744097c7..76e33e1fbc --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@@ -2318,10 -2408,10 +2318,10 @@@ AbortTransaction(void * must be done _before_ releasing locks we hold and _after_ * RecordTransactionAbort. */ - ProcArrayEndTransaction(MyProc, latestXid); + ProcArrayEndTransaction(MyProc); /* - * Post-abort cleanup. See notes in CommitTransaction() concerning + * Post-abort cleanup. See notes in CommitTransaction() concerning * ordering. We can skip all of it if the transaction failed before * creating a resource owner. */ diff --cc src/backend/access/transam/xlog.c index 7865edd82f,3d752c6418..7d8f6dcfd1 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@@ -8619,14 -8643,11 +8628,14 @@@ CreateRestartPoint(int flags ThisTimeLineID = 0; } + /* Real work is done, but log and update before releasing lock. */ + LogCheckpointEnd(true); + /* * Truncate pg_subtrans if possible. We can throw away all data before - * the oldest XMIN of any running transaction. No future transaction will + * the oldest XMIN of any running transaction. No future transaction will * attempt to reference any pg_subtrans entry older than that (see Asserts - * in subtrans.c). When hot standby is disabled, though, we mustn't do + * in subtrans.c). When hot standby is disabled, though, we mustn't do * this because StartupSUBTRANS hasn't been called yet. */ if (EnableHotStandby) diff --cc src/backend/commands/cluster.c index 98a42ec7c3,54a2753182..c5e1ff0c16 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@@ -788,12 -788,12 +788,12 @@@ copy_heap_data(Oid OIDNewHeap, Oid OIDO /* * If the OldHeap has a toast table, get lock on the toast table to keep - * it from being vacuumed. This is needed because autovacuum processes + * it from being vacuumed. This is needed because autovacuum processes * toast tables independently of their main tables, with no lock on the - * latter. If an autovacuum were to start on the toast table after we + * latter. If an autovacuum were to start on the toast table after we - * compute our OldestXmin below, it would use a later OldestXmin, and then + * compute our OldestSnapshot below, it would use a later OldestSnapshot, and then * possibly remove as DEAD toast tuples belonging to main tuples we think - * are only RECENTLY_DEAD. Then we'd fail while trying to copy those + * are only RECENTLY_DEAD. Then we'd fail while trying to copy those * tuples. * * We don't need to open the toast relation here, just lock it. The lock diff --cc src/backend/commands/indexcmds.c index bb0004d9f6,fdfa6ca4f5..d620385800 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@@ -761,10 -761,10 +761,10 @@@ DefineIndex(Oid relationId * Drop the reference snapshot. We must do this before waiting out other * snapshot holders, else we will deadlock against other processes also * doing CREATE INDEX CONCURRENTLY, which would see our snapshot as one - * they must wait for. But first, save the snapshot's snapstholsn to - * they must wait for. But first, save the snapshot's xmin to use as - * limitXmin for GetCurrentVirtualXIDs(). ++ * they must wait for. But first, save the snapshot's snapstholsn to + * pass to GetCurrentVirtualXIDs(). */ - limitXmin = snapshot->xmin; + limitSnapshotLSN = snapshot->snapshotlsn; PopActiveSnapshot(); UnregisterSnapshot(snapshot); diff --cc src/backend/commands/vacuum.c index 3444844302,3d2c73902c..2d37b8d899 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@@ -415,12 -415,11 +415,12 @@@ vacuum_set_xid_limits(Relation rel TransactionId safeLimit; MultiXactId mxactLimit; MultiXactId safeMxactLimit; + TransactionId oldestXmin; /* - * We can always ignore processes running lazy vacuum. This is because we + * We can always ignore processes running lazy vacuum. This is because we * use these values only for deciding which tuples we must keep in the - * tables. Since lazy vacuum doesn't write its XID anywhere, it's safe to + * tables. Since lazy vacuum doesn't write its XID anywhere, it's safe to * ignore it. In theory it could be problematic to ignore lazy vacuums in * a full vacuum, but keep in mind that only one vacuum process can be * working on a particular table at any time, and that each vacuum is diff --cc src/backend/replication/logical/decode.c index 9defde4f3f,7b6114a209..33bd50fe13 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@@ -148,10 -149,10 +148,10 @@@ DecodeXLogOp(LogicalDecodingContext *ct switch (info) { - /* this is also used in END_OF_RECOVERY checkpoints */ + /* this is also used in END_OF_RECOVERY checkpoints */ case XLOG_CHECKPOINT_SHUTDOWN: case XLOG_END_OF_RECOVERY: - SnapBuildSerializationPoint(builder, buf->origptr); + //SnapBuildSerializationPoint(builder, buf->origptr); break; case XLOG_CHECKPOINT_ONLINE: @@@ -180,8 -182,9 +181,8 @@@ static void DecodeXactOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) { - SnapBuild *builder = ctx->snapshot_builder; - XLogRecord *r = &buf->record; + SnapBuild *builder = ctx->snapshot_builder; - ReorderBuffer *reorder = ctx->reorder; + XLogRecord *r = &buf->record; uint8 info = r->xl_info & ~XLR_INFO_MASK; /* no point in doing anything yet, data could not be decoded anyway */ @@@ -272,7 -275,25 +273,8 @@@ break; } - case XLOG_XACT_ASSIGNMENT: - { - xl_xact_assignment *xlrec; - int i; - TransactionId *sub_xid; - - xlrec = (xl_xact_assignment *) buf->record_data; - - sub_xid = &xlrec->xsub[0]; - - for (i = 0; i < xlrec->nsubxacts; i++) - { - ReorderBufferAssignChild(reorder, xlrec->xtop, - *(sub_xid++), buf->origptr); - } - break; - } case XLOG_XACT_PREPARE: + /* * Currently decoding ignores PREPARE TRANSACTION and will just * decode the transaction when the COMMIT PREPARED is sent or diff --cc src/backend/replication/logical/reorderbuffer.c index 304b2569bb,f96e3e1d93..2c88f45986 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@@ -1196,15 -1200,15 +1196,15 @@@ ReorderBufferCopySnap(ReorderBuffer *rb * need to check via cmin/cmax. Thats why we store the toplevel * transaction in there as well. */ - snap->subxip = snap->xip + snap->xcnt; - snap->subxip[i++] = txn->xid; + snap->this_xip = (TransactionId *) (snap + 1); + snap->this_xip[i++] = txn->xid; /* - * nsubxcnt isn't decreased when subtransactions abort, so count - * manually. Since it's an upper boundary it is safe to use it for the - * allocation above. + * nsubxcnt isn't decreased when subtransactions abort, so count manually. + * Since it's an upper boundary it is safe to use it for the allocation + * above. */ - snap->subxcnt = 1; + snap->this_xcnt = 1; dlist_foreach(iter, &txn->subtxns) { @@@ -2944,11 -2971,12 +2944,12 @@@ UpdateLogicalMappings(HTAB *tuplecid_da qsort(files_a, list_length(files), sizeof(RewriteMappingFile *), file_sort_by_lsn); - for(off = 0; off < list_length(files); off++) + for (off = 0; off < list_length(files); off++) { RewriteMappingFile *f = files_a[off]; + elog(DEBUG1, "applying mapping: \"%s\" in %u", f->fname, - snapshot->this_xip[0]); - snapshot->subxip[0]); ++ snapshot->this_xip[0]); ApplyLogicalMappingFile(tuplecid_data, relid, f->fname); pfree(f); } diff --cc src/backend/replication/logical/snapbuild.c index c364c14b8a,f00fd7d422..3b0356fdfc --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@@ -189,8 -244,17 +189,8 @@@ struct SnapBuil * removes knowledge about the previously used resowner, so we save it here. */ ResourceOwner SavedResourceOwnerDuringExport = NULL; - bool ExportInProgress = false; + bool ExportInProgress = false; -/* transaction state manipulation functions */ -static void SnapBuildEndTxn(SnapBuild *builder, XLogRecPtr lsn, TransactionId xid); - -/* ->running manipulation */ -static bool SnapBuildTxnIsRunning(SnapBuild *builder, TransactionId xid); - -/* ->committed manipulation */ -static void SnapBuildPurgeCommittedTxn(SnapBuild *builder); - /* snapshot building/manipulation/distribution functions */ static Snapshot SnapBuildBuildSnapshot(SnapBuild *builder, TransactionId xid); diff --cc src/backend/replication/walsender.c index e95f7965c4,5c11d681c3..c483d41b44 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@@ -1559,21 -1562,21 +1562,22 @@@ ProcessStandbyReplyMessage(void /* compute new replication slot xmin horizon if needed */ static void -PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin) +PhysicalReplicationSlotNewOldestSnapshot(XLogRecPtr feedbackSnapshotLSN) { - bool changed = false; + bool changed = false; volatile ReplicationSlot *slot = MyReplicationSlot; SpinLockAcquire(&slot->mutex); - MyPgXact->xmin = InvalidTransactionId; + MyPgXact->snapshotlsn = InvalidXLogRecPtr; + /* - * For physical replication we don't need the interlock provided - * by xmin and effective_xmin since the consequences of a missed increase - * are limited to query cancellations, so set both at once. + * For physical replication we don't need the interlock provided by xmin + * and effective_xmin since the consequences of a missed increase are + * limited to query cancellations, so set both at once. */ - if (!TransactionIdIsNormal(slot->data.xmin) || - !TransactionIdIsNormal(feedbackXmin) || +#ifdef BROKEN + if (!TransactionIdIsNormal(slot->data.snapshotlsn) || + !TransactionIdIsNormal(feedback) || TransactionIdPrecedes(slot->data.xmin, feedbackXmin)) { changed = true; @@@ -1647,17 -1671,14 +1651,17 @@@ ProcessStandbyHSFeedbackMessage(void * * If we're using a replication slot we reserve the xmin via that, * otherwise via the walsender's PGXACT entry. - + * * XXX: It might make sense to introduce ephemeral slots and always use * the slot mechanism. + * + * FIXME: setting an LSN is not atomic... */ + MyPgXact->snapshotlsn = feedbackLSN; - if (MyReplicationSlot != NULL) /* XXX: persistency configurable? */ + if (MyReplicationSlot != NULL) /* XXX: persistency configurable? */ - PhysicalReplicationSlotNewXmin(feedbackXmin); + PhysicalReplicationSlotNewOldestSnapshot(feedbackLSN); else - MyPgXact->xmin = feedbackXmin; + MyPgXact->snapshotlsn = feedbackLSN; } /* diff --cc src/backend/storage/ipc/procarray.c index 8371f6f5cb,cdd92d99a2..f254095f21 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@@ -422,8 -686,49 +422,8 @@@ ProcArrayApplyRecoveryInfo(RunningTrans ExtendSUBTRANS(latestObservedXid); TransactionIdAdvance(latestObservedXid); } - TransactionIdRetreat(latestObservedXid); /* = running->nextXid - 1 */ + TransactionIdRetreat(latestObservedXid); /* = running->nextXid - 1 */ - /* ---------- - * Now we've got the running xids we need to set the global values that - * are used to track snapshots as they evolve further. - * - * - latestCompletedXid which will be the xmax for snapshots - * - lastOverflowedXid which shows whether snapshots overflow - * - nextXid - * - * If the snapshot overflowed, then we still initialise with what we know, - * but the recovery snapshot isn't fully valid yet because we know there - * are some subxids missing. We don't know the specific subxids that are - * missing, so conservatively assume the last one is latestObservedXid. - * ---------- - */ - if (running->subxid_overflow) - { - standbyState = STANDBY_SNAPSHOT_PENDING; - - standbySnapshotPendingXmin = latestObservedXid; - procArray->lastOverflowedXid = latestObservedXid; - } - else - { - standbyState = STANDBY_SNAPSHOT_READY; - - standbySnapshotPendingXmin = InvalidTransactionId; - } - - /* - * If a transaction wrote a commit record in the gap between taking and - * logging the snapshot then latestCompletedXid may already be higher than - * the value from the snapshot, so check before we use the incoming value. - */ - if (TransactionIdPrecedes(ShmemVariableCache->latestCompletedXid, - running->latestCompletedXid)) - ShmemVariableCache->latestCompletedXid = running->latestCompletedXid; - - Assert(TransactionIdIsNormal(ShmemVariableCache->latestCompletedXid)); - - LWLockRelease(ProcArrayLock); - /* * ShmemVariableCache->nextXid must be beyond any observed xid. * diff --cc src/backend/utils/time/tqual.c index b37dbebda3,75cd53e1fd..06d7b86ea1 --- a/src/backend/utils/time/tqual.c +++ b/src/backend/utils/time/tqual.c @@@ -1452,11 -1605,10 +1453,11 @@@ HeapTupleSatisfiesHistoricMVCC(HeapTupl /* * another transaction might have (tried to) delete this tuple or - * cmin/cmax was stored in a combocid. So we need to lookup the - * actual values externally. + * cmin/cmax was stored in a combocid. So we need to lookup the actual + * values externally. */ - resolved = ResolveCminCmaxDuringDecoding(HistoricSnapshotGetTupleCids(), snapshot, + resolved = ResolveCminCmaxDuringDecoding(HistoricSnapshotGetTupleCids(), + snapshot, htup, buffer, &cmin, &cmax); @@@ -1466,14 -1618,37 +1467,14 @@@ Assert(cmin != InvalidCommandId); if (cmin >= snapshot->curcid) - return false; /* inserted after scan started */ + return false; /* inserted after scan started */ /* fall through */ } - /* committed before our xmin horizon. Do a normal visibility check. */ - else if (TransactionIdPrecedes(xmin, snapshot->xmin)) - { - Assert(!(HeapTupleHeaderXminCommitted(tuple) && - !TransactionIdDidCommit(xmin))); - - /* check for hint bit first, consult clog afterwards */ - if (!HeapTupleHeaderXminCommitted(tuple) && - !TransactionIdDidCommit(xmin)) - return false; - /* fall through */ - } - /* beyond our xmax horizon, i.e. invisible */ - else if (TransactionIdFollowsOrEquals(xmin, snapshot->xmax)) - { - return false; - } - /* check if it's a committed transaction in [xmin, xmax) */ - else if (TransactionIdInArray(xmin, snapshot->xip, snapshot->xcnt)) - { - /* fall through */ - } - /* - * none of the above, i.e. between [xmin, xmax) but hasn't committed. I.e. - * invisible. + * it's not "this" transaction. Do a normal visibility check using the + * snapshot. */ - else + else if (!XidVisibleInSnapshot(xmin, snapshot, false, &hintstatus)) { return false; } @@@ -1496,15 -1672,14 +1498,15 @@@ } /* check if it's one of our txids, toplevel is also in there */ - if (TransactionIdInArray(xmax, snapshot->subxip, snapshot->subxcnt)) + if (TransactionIdInArray(xmax, snapshot->this_xip, snapshot->this_xcnt)) { - bool resolved; - CommandId cmin; - CommandId cmax = HeapTupleHeaderGetRawCommandId(tuple); + bool resolved; + CommandId cmin; + CommandId cmax = HeapTupleHeaderGetRawCommandId(tuple); /* Lookup actual cmin/cmax values */ - resolved = ResolveCminCmaxDuringDecoding(HistoricSnapshotGetTupleCids(), snapshot, + resolved = ResolveCminCmaxDuringDecoding(HistoricSnapshotGetTupleCids(), + snapshot, htup, buffer, &cmin, &cmax); @@@ -1514,16 -1689,30 +1516,16 @@@ Assert(cmax != InvalidCommandId); if (cmax >= snapshot->curcid) - return true; /* deleted after scan started */ + return true; /* deleted after scan started */ else - return false; /* deleted before scan started */ + return false; /* deleted before scan started */ } - /* below xmin horizon, normal transaction state is valid */ - else if (TransactionIdPrecedes(xmax, snapshot->xmin)) - { - Assert(!(tuple->t_infomask & HEAP_XMAX_COMMITTED && - !TransactionIdDidCommit(xmax))); - - /* check hint bit first */ - if (tuple->t_infomask & HEAP_XMAX_COMMITTED) - return false; - - /* check clog */ - return !TransactionIdDidCommit(xmax); - } - /* above xmax horizon, we cannot possibly see the deleting transaction */ - else if (TransactionIdFollowsOrEquals(xmax, snapshot->xmax)) - return true; - /* xmax is between [xmin, xmax), check known committed array */ - else if (TransactionIdInArray(xmax, snapshot->xip, snapshot->xcnt)) + /* + * it's not "this" transaction. Do a normal visibility check using the + * snapshot. + */ + if (XidVisibleInSnapshot(xmax, snapshot, false, &hintstatus)) return false; - /* xmax is between [xmin, xmax), but known not to have committed yet */ else return true; } diff --cc src/include/pgstat.h index 70908a1fe1,d9de09fea0..9b79916b34 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@@ -758,10 -759,10 +760,10 @@@ typedef struct LocalPgBackendStatu TransactionId backend_xid; /* - * The oldest snapshot of the current session if available, InvalidXLogRecPtr - * if not. - * The xmin of the current session if available, InvalidTransactionId if - * not. ++ * The oldest snapshot of the current session if available, ++ * InvalidXLogRecPtr if not. */ - TransactionId backend_xmin; + XLogRecPtr backend_snapshot; } LocalPgBackendStatus; /* diff --cc src/include/replication/snapbuild.h index be3b35d145,e5d61ff3c4..02acdb3042 --- a/src/include/replication/snapbuild.h +++ b/src/include/replication/snapbuild.h @@@ -41,8 -51,10 +41,8 @@@ struct ReorderBuffer struct xl_heap_new_cid; struct xl_running_xacts; -extern void CheckPointSnapBuild(void); - extern SnapBuild *AllocateSnapshotBuilder(struct ReorderBuffer *cache, - TransactionId xmin_horizon, XLogRecPtr start_lsn); + TransactionId xmin_horizon, XLogRecPtr start_lsn); extern void FreeSnapshotBuilder(SnapBuild *cache); extern void SnapBuildSnapDecRefcount(Snapshot snap); @@@ -55,18 -67,17 +55,18 @@@ extern SnapBuildState SnapBuildCurrentS extern bool SnapBuildXactNeedsSkip(SnapBuild *snapstate, XLogRecPtr ptr); extern void SnapBuildCommitTxn(SnapBuild *builder, XLogRecPtr lsn, - TransactionId xid, int nsubxacts, - TransactionId *subxacts); + TransactionId xid, int nsubxacts, + TransactionId *subxacts); extern void SnapBuildAbortTxn(SnapBuild *builder, XLogRecPtr lsn, - TransactionId xid, int nsubxacts, - TransactionId *subxacts); + TransactionId xid, int nsubxacts, + TransactionId *subxacts); extern bool SnapBuildProcessChange(SnapBuild *builder, TransactionId xid, - XLogRecPtr lsn); + XLogRecPtr lsn); extern void SnapBuildProcessNewCid(SnapBuild *builder, TransactionId xid, - XLogRecPtr lsn, struct xl_heap_new_cid *cid); + XLogRecPtr lsn, struct xl_heap_new_cid *cid); extern void SnapBuildProcessRunningXacts(SnapBuild *builder, XLogRecPtr lsn, - struct xl_running_xacts *running); + struct xl_running_xacts *running); -extern void SnapBuildSerializationPoint(SnapBuild *builder, XLogRecPtr lsn); +extern void SnapBuildProcessInitialSnapshot(SnapBuild *builder, XLogRecPtr lsn, - TransactionId xmin, TransactionId xmax); ++ TransactionId xmin, TransactionId xmax); #endif /* SNAPBUILD_H */ diff --cc src/include/storage/proc.h index a0916aec63,c23f4da5b6..62074a3ae1 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@@ -112,7 -131,9 +113,7 @@@ struct PGPRO */ SHM_QUEUE myProcLocks[NUM_LOCK_PARTITIONS]; - /* Per-backend LWLock. Protects fields below. */ - struct XidCache subxids; /* cache for subtransaction XIDs */ - + /* Per-backend LWLock. Protects fields below. */ LWLock *backendLock; /* protects the fields below */ /* Lock manager data, recording fast-path locks taken by this backend. */