From e5f3839af685c303d8ebcc1ea0d407c124372931 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 23 Dec 2025 07:41:34 +0900 Subject: [PATCH] Switch buffile.c/h to use pgoff_t instead of off_t off_t was previously used for offsets, which is 4 bytes on Windows, hence limiting the backend code to a hard limit for files longer than 2GB. This leads to some simplification in these files, removing some casts based on long, also 4 bytes on Windows. This commit removes one comment introduced in db3c4c3a2d98, not relevant anymore as pgoff_t is a safe 8-byte alternative on Windows. This change is surprisingly not invasive, as the callers of BufFileTell(), BufFileSeek() and BufFileTruncateFileSet() (worker.c, tuplestore.c, etc.) track offsets in local structures that just to switch from off_t to pgoff_t for the most part. The file is still relying on a maximum file size of MAX_PHYSICAL_FILESIZE (1GB). This change allows the code to make this maximum potentially larger in the future, or larger on a per-demand basis. Reviewed-by: Chao Li Discussion: https://postgr.es/m/aUStrqoOCDRFAq1M@paquier.xyz --- src/backend/replication/logical/worker.c | 8 ++++---- src/backend/storage/file/buffile.c | 23 +++++++++++------------ src/backend/utils/sort/tuplestore.c | 12 ++++++------ src/include/storage/buffile.h | 6 +++--- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index fc64476a9ef..d1ee0261c64 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -530,7 +530,7 @@ typedef struct SubXactInfo { TransactionId xid; /* XID of the subxact */ int fileno; /* file number in the buffile */ - off_t offset; /* offset in the file */ + pgoff_t offset; /* offset in the file */ } SubXactInfo; /* Sub-transaction data for the current streaming transaction */ @@ -2226,12 +2226,12 @@ apply_handle_stream_abort(StringInfo s) */ static void ensure_last_message(FileSet *stream_fileset, TransactionId xid, int fileno, - off_t offset) + pgoff_t offset) { char path[MAXPGPATH]; BufFile *fd; int last_fileno; - off_t last_offset; + pgoff_t last_offset; Assert(!IsTransactionState()); @@ -2266,7 +2266,7 @@ apply_spooled_messages(FileSet *stream_fileset, TransactionId xid, MemoryContext oldcxt; ResourceOwner oldowner; int fileno; - off_t offset; + pgoff_t offset; if (!am_parallel_apply_worker()) maybe_start_skipping_changes(lsn); diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 4e520065ae0..85b316d879d 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -92,7 +92,7 @@ struct BufFile * Position as seen by user of BufFile is (curFile, curOffset + pos). */ int curFile; /* file index (0..n) part of current pos */ - off_t curOffset; /* offset part of current pos */ + pgoff_t curOffset; /* offset part of current pos */ int pos; /* next read/write position in buffer */ int nbytes; /* total # of valid bytes in buffer */ @@ -503,7 +503,7 @@ BufFileDumpBuffer(BufFile *file) */ while (wpos < file->nbytes) { - off_t availbytes; + pgoff_t availbytes; instr_time io_start; instr_time io_time; @@ -524,7 +524,7 @@ BufFileDumpBuffer(BufFile *file) bytestowrite = file->nbytes - wpos; availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset; - if ((off_t) bytestowrite > availbytes) + if ((pgoff_t) bytestowrite > availbytes) bytestowrite = (int) availbytes; thisfile = file->files[file->curFile]; @@ -729,7 +729,7 @@ BufFileFlush(BufFile *file) * BufFileSeek * * Like fseek(), except that target position needs two values in order to - * work when logical filesize exceeds maximum value representable by off_t. + * work when logical filesize exceeds maximum value representable by pgoff_t. * We do not support relative seeks across more than that, however. * I/O errors are reported by ereport(). * @@ -737,10 +737,10 @@ BufFileFlush(BufFile *file) * impossible seek is attempted. */ int -BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) +BufFileSeek(BufFile *file, int fileno, pgoff_t offset, int whence) { int newFile; - off_t newOffset; + pgoff_t newOffset; switch (whence) { @@ -754,8 +754,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) /* * Relative seek considers only the signed offset, ignoring - * fileno. Note that large offsets (> 1 GB) risk overflow in this - * add, unless we have 64-bit off_t. + * fileno. */ newFile = file->curFile; newOffset = (file->curOffset + file->pos) + offset; @@ -830,7 +829,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) } void -BufFileTell(BufFile *file, int *fileno, off_t *offset) +BufFileTell(BufFile *file, int *fileno, pgoff_t *offset) { *fileno = file->curFile; *offset = file->curOffset + file->pos; @@ -852,7 +851,7 @@ BufFileSeekBlock(BufFile *file, int64 blknum) { return BufFileSeek(file, (int) (blknum / BUFFILE_SEG_SIZE), - (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ, + (pgoff_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ, SEEK_SET); } @@ -925,11 +924,11 @@ BufFileAppend(BufFile *target, BufFile *source) * and the offset. */ void -BufFileTruncateFileSet(BufFile *file, int fileno, off_t offset) +BufFileTruncateFileSet(BufFile *file, int fileno, pgoff_t offset) { int numFiles = file->numFiles; int newFile = fileno; - off_t newOffset = file->curOffset; + pgoff_t newOffset = file->curOffset; char segment_name[MAXPGPATH]; int i; diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c index def945b0454..9701b813360 100644 --- a/src/backend/utils/sort/tuplestore.c +++ b/src/backend/utils/sort/tuplestore.c @@ -94,7 +94,7 @@ typedef struct bool eof_reached; /* read has reached EOF */ int current; /* next array index to read */ int file; /* temp file# */ - off_t offset; /* byte offset in file */ + pgoff_t offset; /* byte offset in file */ } TSReadPointer; /* @@ -179,7 +179,7 @@ struct Tuplestorestate int readptrsize; /* allocated length of readptrs array */ int writepos_file; /* file# (valid if READFILE state) */ - off_t writepos_offset; /* offset (valid if READFILE state) */ + pgoff_t writepos_offset; /* offset (valid if READFILE state) */ }; #define COPYTUP(state,tup) ((*(state)->copytup) (state, tup)) @@ -1051,7 +1051,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward, * Back up to fetch previously-returned tuple's ending length * word. If seek fails, assume we are at start of file. */ - if (BufFileSeek(state->myfile, 0, -(long) sizeof(unsigned int), + if (BufFileSeek(state->myfile, 0, -(pgoff_t) sizeof(unsigned int), SEEK_CUR) != 0) { /* even a failed backwards fetch gets you out of eof state */ @@ -1072,7 +1072,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward, * Back up to get ending length word of tuple before it. */ if (BufFileSeek(state->myfile, 0, - -(long) (tuplen + 2 * sizeof(unsigned int)), + -(pgoff_t) (tuplen + 2 * sizeof(unsigned int)), SEEK_CUR) != 0) { /* @@ -1082,7 +1082,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward, * what in-memory case does). */ if (BufFileSeek(state->myfile, 0, - -(long) (tuplen + sizeof(unsigned int)), + -(pgoff_t) (tuplen + sizeof(unsigned int)), SEEK_CUR) != 0) ereport(ERROR, (errcode_for_file_access(), @@ -1099,7 +1099,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward, * length word of the tuple, so back up to that point. */ if (BufFileSeek(state->myfile, 0, - -(long) tuplen, + -(pgoff_t) tuplen, SEEK_CUR) != 0) ereport(ERROR, (errcode_for_file_access(), diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h index a2f4821f240..4908dc56c7b 100644 --- a/src/include/storage/buffile.h +++ b/src/include/storage/buffile.h @@ -42,8 +42,8 @@ pg_nodiscard extern size_t BufFileRead(BufFile *file, void *ptr, size_t size); extern void BufFileReadExact(BufFile *file, void *ptr, size_t size); extern size_t BufFileReadMaybeEOF(BufFile *file, void *ptr, size_t size, bool eofOK); extern void BufFileWrite(BufFile *file, const void *ptr, size_t size); -extern int BufFileSeek(BufFile *file, int fileno, off_t offset, int whence); -extern void BufFileTell(BufFile *file, int *fileno, off_t *offset); +extern int BufFileSeek(BufFile *file, int fileno, pgoff_t offset, int whence); +extern void BufFileTell(BufFile *file, int *fileno, pgoff_t *offset); extern int BufFileSeekBlock(BufFile *file, int64 blknum); extern int64 BufFileSize(BufFile *file); extern int64 BufFileAppend(BufFile *target, BufFile *source); @@ -54,6 +54,6 @@ extern BufFile *BufFileOpenFileSet(FileSet *fileset, const char *name, int mode, bool missing_ok); extern void BufFileDeleteFileSet(FileSet *fileset, const char *name, bool missing_ok); -extern void BufFileTruncateFileSet(BufFile *file, int fileno, off_t offset); +extern void BufFileTruncateFileSet(BufFile *file, int fileno, pgoff_t offset); #endif /* BUFFILE_H */ -- 2.39.5