diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml index 61c12975e4ad..16b50b5a4df0 100644 --- a/doc/src/sgml/ref/pg_verifybackup.sgml +++ b/doc/src/sgml/ref/pg_verifybackup.sgml @@ -36,10 +36,7 @@ PostgreSQL documentation backup_manifest generated by the server at the time of the backup. The backup may be stored either in the "plain" or the "tar" format; this includes tar-format backups compressed with any algorithm - supported by pg_basebackup. However, at present, - WAL verification is supported only for plain-format - backups. Therefore, if the backup is stored in tar-format, the - -n, --no-parse-wal option should be used. + supported by pg_basebackup. @@ -261,7 +258,7 @@ PostgreSQL documentation - + Try to parse WAL files stored in the specified directory, rather than diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml index ce23add5577e..25f96f86168a 100644 --- a/doc/src/sgml/ref/pg_waldump.sgml +++ b/doc/src/sgml/ref/pg_waldump.sgml @@ -141,13 +141,21 @@ PostgreSQL documentation - Specifies a directory to search for WAL segment files or a - directory with a pg_wal subdirectory that + Specifies a tar archive or a directory to search for WAL segment files + or a directory with a pg_wal subdirectory that contains such files. The default is to search in the current directory, the pg_wal subdirectory of the current directory, and the pg_wal subdirectory of PGDATA. + + If a tar archive is provided and its WAL segment files are not in + sequential order, those files will be written to a temporary directory + named starting with waldump/. This directory will be + created inside the directory specified by the TMPDIR + environment variable if it is set; otherwise, it will be created within + the same directory as the tar archive. + diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c index c9b24df7c053..237262bd5332 100644 --- a/src/bin/pg_verifybackup/pg_verifybackup.c +++ b/src/bin/pg_verifybackup/pg_verifybackup.c @@ -74,7 +74,9 @@ pg_noreturn static void report_manifest_error(JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2, 3); -static void verify_tar_backup(verifier_context *context, DIR *dir); +static void verify_tar_backup(verifier_context *context, DIR *dir, + char **base_archive_path, + char **wal_archive_path); static void verify_plain_backup_directory(verifier_context *context, char *relpath, char *fullpath, DIR *dir); @@ -83,7 +85,9 @@ static void verify_plain_backup_file(verifier_context *context, char *relpath, static void verify_control_file(const char *controlpath, uint64 manifest_system_identifier); static void precheck_tar_backup_file(verifier_context *context, char *relpath, - char *fullpath, SimplePtrList *tarfiles); + char *fullpath, SimplePtrList *tarfiles, + char **base_archive_path, + char **wal_archive_path); static void verify_tar_file(verifier_context *context, char *relpath, char *fullpath, astreamer *streamer); static void report_extra_backup_files(verifier_context *context); @@ -93,7 +97,7 @@ static void verify_file_checksum(verifier_context *context, uint8 *buffer); static void parse_required_wal(verifier_context *context, char *pg_waldump_path, - char *wal_directory); + char *wal_path); static astreamer *create_archive_verifier(verifier_context *context, char *archive_name, Oid tblspc_oid, @@ -126,7 +130,7 @@ main(int argc, char **argv) {"progress", no_argument, NULL, 'P'}, {"quiet", no_argument, NULL, 'q'}, {"skip-checksums", no_argument, NULL, 's'}, - {"wal-directory", required_argument, NULL, 'w'}, + {"wal-path", required_argument, NULL, 'w'}, {NULL, 0, NULL, 0} }; @@ -135,7 +139,9 @@ main(int argc, char **argv) char *manifest_path = NULL; bool no_parse_wal = false; bool quiet = false; - char *wal_directory = NULL; + char *wal_path = NULL; + char *base_archive_path = NULL; + char *wal_archive_path = NULL; char *pg_waldump_path = NULL; DIR *dir; @@ -221,8 +227,8 @@ main(int argc, char **argv) context.skip_checksums = true; break; case 'w': - wal_directory = pstrdup(optarg); - canonicalize_path(wal_directory); + wal_path = pstrdup(optarg); + canonicalize_path(wal_path); break; default: /* getopt_long already emitted a complaint */ @@ -285,10 +291,6 @@ main(int argc, char **argv) manifest_path = psprintf("%s/backup_manifest", context.backup_directory); - /* By default, look for the WAL in the backup directory, too. */ - if (wal_directory == NULL) - wal_directory = psprintf("%s/pg_wal", context.backup_directory); - /* * Try to read the manifest. We treat any errors encountered while parsing * the manifest as fatal; there doesn't seem to be much point in trying to @@ -331,17 +333,6 @@ main(int argc, char **argv) pfree(path); } - /* - * XXX: In the future, we should consider enhancing pg_waldump to read WAL - * files from an archive. - */ - if (!no_parse_wal && context.format == 't') - { - pg_log_error("pg_waldump cannot read tar files"); - pg_log_error_hint("You must use -n/--no-parse-wal when verifying a tar-format backup."); - exit(1); - } - /* * Perform the appropriate type of verification appropriate based on the * backup format. This will close 'dir'. @@ -350,7 +341,7 @@ main(int argc, char **argv) verify_plain_backup_directory(&context, NULL, context.backup_directory, dir); else - verify_tar_backup(&context, dir); + verify_tar_backup(&context, dir, &base_archive_path, &wal_archive_path); /* * The "matched" flag should now be set on every entry in the hash table. @@ -368,12 +359,35 @@ main(int argc, char **argv) if (context.format == 'p' && !context.skip_checksums) verify_backup_checksums(&context); + /* + * By default, WAL files are expected to be found in the backup directory + * for plain-format backups. In the case of tar-format backups, if a + * separate WAL archive is not found, the WAL files are most likely + * included within the main data directory archive. + */ + if (wal_path == NULL) + { + if (context.format == 'p') + wal_path = psprintf("%s/pg_wal", context.backup_directory); + else if (wal_archive_path) + wal_path = wal_archive_path; + else if (base_archive_path) + wal_path = base_archive_path; + else + { + pg_log_error("WAL archive not found"); + pg_log_error_hint("Specify the correct path using the option -w/--wal-path. " + "Or you must use -n/--no-parse-wal when verifying a tar-format backup."); + exit(1); + } + } + /* * Try to parse the required ranges of WAL records, unless we were told * not to do so. */ if (!no_parse_wal) - parse_required_wal(&context, pg_waldump_path, wal_directory); + parse_required_wal(&context, pg_waldump_path, wal_path); /* * If everything looks OK, tell the user this, unless we were asked to @@ -787,7 +801,8 @@ verify_control_file(const char *controlpath, uint64 manifest_system_identifier) * close when we're done with it. */ static void -verify_tar_backup(verifier_context *context, DIR *dir) +verify_tar_backup(verifier_context *context, DIR *dir, char **base_archive_path, + char **wal_archive_path) { struct dirent *dirent; SimplePtrList tarfiles = {NULL, NULL}; @@ -816,7 +831,8 @@ verify_tar_backup(verifier_context *context, DIR *dir) char *fullpath; fullpath = psprintf("%s/%s", context->backup_directory, filename); - precheck_tar_backup_file(context, filename, fullpath, &tarfiles); + precheck_tar_backup_file(context, filename, fullpath, &tarfiles, + base_archive_path, wal_archive_path); pfree(fullpath); } } @@ -875,11 +891,13 @@ verify_tar_backup(verifier_context *context, DIR *dir) * * The arguments to this function are mostly the same as the * verify_plain_backup_file. The additional argument outputs a list of valid - * tar files. + * tar files, along with the full paths to the main archive and the WAL + * directory archive. */ static void precheck_tar_backup_file(verifier_context *context, char *relpath, - char *fullpath, SimplePtrList *tarfiles) + char *fullpath, SimplePtrList *tarfiles, + char **base_archive_path, char **wal_archive_path) { struct stat sb; Oid tblspc_oid = InvalidOid; @@ -918,9 +936,17 @@ precheck_tar_backup_file(verifier_context *context, char *relpath, * extension such as .gz, .lz4, or .zst. */ if (strncmp("base", relpath, 4) == 0) + { suffix = relpath + 4; + + *base_archive_path = pstrdup(fullpath); + } else if (strncmp("pg_wal", relpath, 6) == 0) + { suffix = relpath + 6; + + *wal_archive_path = pstrdup(fullpath); + } else { /* Expected a .tar file here. */ @@ -1198,7 +1224,7 @@ verify_file_checksum(verifier_context *context, manifest_file *m, */ static void parse_required_wal(verifier_context *context, char *pg_waldump_path, - char *wal_directory) + char *wal_path) { manifest_data *manifest = context->manifest; manifest_wal_range *this_wal_range = manifest->first_wal_range; @@ -1208,7 +1234,7 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path, char *pg_waldump_cmd; pg_waldump_cmd = psprintf("\"%s\" --quiet --path=\"%s\" --timeline=%u --start=%X/%08X --end=%X/%08X\n", - pg_waldump_path, wal_directory, this_wal_range->tli, + pg_waldump_path, wal_path, this_wal_range->tli, LSN_FORMAT_ARGS(this_wal_range->start_lsn), LSN_FORMAT_ARGS(this_wal_range->end_lsn)); fflush(NULL); @@ -1376,7 +1402,7 @@ usage(void) printf(_(" -P, --progress show progress information\n")); printf(_(" -q, --quiet do not print any output, except for errors\n")); printf(_(" -s, --skip-checksums skip checksum verification\n")); - printf(_(" -w, --wal-directory=PATH use specified path for WAL files\n")); + printf(_(" -w, --wal-path=PATH use specified path for WAL files\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); diff --git a/src/bin/pg_verifybackup/t/002_algorithm.pl b/src/bin/pg_verifybackup/t/002_algorithm.pl index ae16c11bc4dd..4f284a9e8285 100644 --- a/src/bin/pg_verifybackup/t/002_algorithm.pl +++ b/src/bin/pg_verifybackup/t/002_algorithm.pl @@ -30,10 +30,6 @@ sub test_checksums { # Add switch to get a tar-format backup push @backup, ('--format' => 'tar'); - - # Add switch to skip WAL verification, which is not yet supported for - # tar-format backups - push @verify, ('--no-parse-wal'); } # A backup with a bogus algorithm should fail. diff --git a/src/bin/pg_verifybackup/t/003_corruption.pl b/src/bin/pg_verifybackup/t/003_corruption.pl index 1dd60f709cf6..f1ebdbb46b4b 100644 --- a/src/bin/pg_verifybackup/t/003_corruption.pl +++ b/src/bin/pg_verifybackup/t/003_corruption.pl @@ -193,10 +193,8 @@ command_ok([ $tar, '-cf' => "$tar_backup_path/base.tar", '.' ]); chdir($cwd) || die "chdir: $!"; - # Now check that the backup no longer verifies. We must use -n - # here, because pg_waldump can't yet read WAL from a tarfile. command_fails_like( - [ 'pg_verifybackup', '--no-parse-wal', $tar_backup_path ], + [ 'pg_verifybackup', $tar_backup_path ], $scenario->{'fails_like'}, "corrupt backup fails verification: $name"); diff --git a/src/bin/pg_verifybackup/t/007_wal.pl b/src/bin/pg_verifybackup/t/007_wal.pl index babc4f0a86b8..b07f80719b04 100644 --- a/src/bin/pg_verifybackup/t/007_wal.pl +++ b/src/bin/pg_verifybackup/t/007_wal.pl @@ -42,10 +42,10 @@ command_ok( [ 'pg_verifybackup', - '--wal-directory' => $relocated_pg_wal, + '--wal-path' => $relocated_pg_wal, $backup_path ], - '--wal-directory can be used to specify WAL directory'); + '--wal-path can be used to specify WAL directory'); # Move directory back to original location. rename($relocated_pg_wal, $original_pg_wal) || die "rename pg_wal back: $!"; diff --git a/src/bin/pg_verifybackup/t/008_untar.pl b/src/bin/pg_verifybackup/t/008_untar.pl index bc3d6b352ad5..09079a94feed 100644 --- a/src/bin/pg_verifybackup/t/008_untar.pl +++ b/src/bin/pg_verifybackup/t/008_untar.pl @@ -47,7 +47,6 @@ SELECT oid FROM pg_tablespace WHERE spcname = 'regress_ts1')); my $backup_path = $primary->backup_dir . '/server-backup'; -my $extract_path = $primary->backup_dir . '/extracted-backup'; my @test_configuration = ( { @@ -123,14 +122,12 @@ # Verify tar backup. $primary->command_ok( [ - 'pg_verifybackup', '--no-parse-wal', - '--exit-on-error', $backup_path, + 'pg_verifybackup', '--exit-on-error', $backup_path, ], "verify backup, compression $method"); # Cleanup. rmtree($backup_path); - rmtree($extract_path); } } diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl index b62faeb5acfa..5b0e76ee69d1 100644 --- a/src/bin/pg_verifybackup/t/010_client_untar.pl +++ b/src/bin/pg_verifybackup/t/010_client_untar.pl @@ -32,7 +32,6 @@ close $jf; my $backup_path = $primary->backup_dir . '/client-backup'; -my $extract_path = $primary->backup_dir . '/extracted-backup'; my @test_configuration = ( { @@ -137,13 +136,11 @@ # Verify tar backup. $primary->command_ok( [ - 'pg_verifybackup', '--no-parse-wal', - '--exit-on-error', $backup_path, + 'pg_verifybackup', '--exit-on-error', $backup_path, ], "verify backup, compression $method"); # Cleanup. - rmtree($extract_path); rmtree($backup_path); } } diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index 4c1ee649501f..aabb87566a2d 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -3,6 +3,9 @@ PGFILEDESC = "pg_waldump - decode and display WAL" PGAPPICON=win32 +# make these available to TAP test scripts +export TAR + subdir = src/bin/pg_waldump top_builddir = ../../.. include $(top_builddir)/src/Makefile.global @@ -10,13 +13,15 @@ include $(top_builddir)/src/Makefile.global OBJS = \ $(RMGRDESCOBJS) \ $(WIN32RES) \ + archive_waldump.o \ compat.o \ pg_waldump.o \ rmgrdesc.o \ xlogreader.o \ xlogstats.o -override CPPFLAGS := -DFRONTEND $(CPPFLAGS) +override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) +LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils RMGRDESCSOURCES = $(sort $(notdir $(wildcard $(top_srcdir)/src/backend/access/rmgrdesc/*desc*.c))) RMGRDESCOBJS = $(patsubst %.c,%.o,$(RMGRDESCSOURCES)) diff --git a/src/bin/pg_waldump/archive_waldump.c b/src/bin/pg_waldump/archive_waldump.c new file mode 100644 index 000000000000..2038876a5165 --- /dev/null +++ b/src/bin/pg_waldump/archive_waldump.c @@ -0,0 +1,789 @@ +/*------------------------------------------------------------------------- + * + * archive_waldump.c + * A generic facility for reading WAL data from tar archives via archive + * streamer. + * + * Portions Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_waldump/archive_waldump.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include + +#include "access/xlog_internal.h" +#include "common/file_perm.h" +#include "common/hashfn.h" +#include "common/logging.h" +#include "fe_utils/simple_list.h" +#include "pg_waldump.h" + +/* + * How many bytes should we try to read from a file at once? + */ +#define READ_CHUNK_SIZE (128 * 1024) + +/* Temporary exported WAL file directory */ +static char *TmpWalSegDir = NULL; + +/* Structure for storing the WAL segment data from the archive */ +typedef struct ArchivedWALEntry +{ + uint32 status; /* hash status */ + XLogSegNo segno; /* hash key: WAL segment number */ + TimeLineID timeline; /* timeline of this wal file */ + + StringInfoData buf; + bool tmpseg_exists; /* spill file exists? */ + + int total_read; /* total read of this WAL segment, including + * buffered and temporarily written data */ +} ArchivedWALEntry; + +#define SH_PREFIX ArchivedWAL +#define SH_ELEMENT_TYPE ArchivedWALEntry +#define SH_KEY_TYPE XLogSegNo +#define SH_KEY segno +#define SH_HASH_KEY(tb, key) murmurhash64((uint64) key) +#define SH_EQUAL(tb, a, b) (a == b) +#define SH_GET_HASH(tb, a) a->hash +#define SH_SCOPE static inline +#define SH_RAW_ALLOCATOR pg_malloc0 +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +static ArchivedWAL_hash *ArchivedWAL_HTAB = NULL; + +typedef struct astreamer_waldump +{ + astreamer base; + XLogDumpPrivate *privateInfo; +} astreamer_waldump; + +static int read_archive_file(XLogDumpPrivate *privateInfo, Size count); +static ArchivedWALEntry *get_archive_wal_entry(XLogSegNo segno, + XLogDumpPrivate *privateInfo); +static void setup_tmpseg_dir(const char *waldir); +static void cleanup_tmpseg_dir_atexit(void); + +static FILE *prepare_tmp_write(XLogSegNo segno); +static void perform_tmp_write(XLogSegNo segno, StringInfo buf, FILE *file); + +static astreamer *astreamer_waldump_new(XLogDumpPrivate *privateInfo); +static void astreamer_waldump_content(astreamer *streamer, + astreamer_member *member, + const char *data, int len, + astreamer_archive_context context); +static void astreamer_waldump_finalize(astreamer *streamer); +static void astreamer_waldump_free(astreamer *streamer); + +static bool member_is_wal_file(astreamer_waldump *mystreamer, + astreamer_member *member, + XLogSegNo *curSegNo, + TimeLineID *curTimeline); + +static const astreamer_ops astreamer_waldump_ops = { + .content = astreamer_waldump_content, + .finalize = astreamer_waldump_finalize, + .free = astreamer_waldump_free +}; + +/* + * Returns true if the given file is a tar archive and outputs its compression + * algorithm. + */ +bool +is_archive_file(const char *fname, pg_compress_algorithm *compression) +{ + int fname_len = strlen(fname); + pg_compress_algorithm compress_algo; + + /* Now, check the compression type of the tar */ + if (fname_len > 4 && + strcmp(fname + fname_len - 4, ".tar") == 0) + compress_algo = PG_COMPRESSION_NONE; + else if (fname_len > 4 && + strcmp(fname + fname_len - 4, ".tgz") == 0) + compress_algo = PG_COMPRESSION_GZIP; + else if (fname_len > 7 && + strcmp(fname + fname_len - 7, ".tar.gz") == 0) + compress_algo = PG_COMPRESSION_GZIP; + else if (fname_len > 8 && + strcmp(fname + fname_len - 8, ".tar.lz4") == 0) + compress_algo = PG_COMPRESSION_LZ4; + else if (fname_len > 8 && + strcmp(fname + fname_len - 8, ".tar.zst") == 0) + compress_algo = PG_COMPRESSION_ZSTD; + else + return false; + + *compression = compress_algo; + + return true; +} + +/* + * Initializes the tar archive reader, creates a hash table for WAL entries, + * checks for existing valid WAL segments in the archive file and retrieves the + * segment size, and sets up filters for relevant entries. It also configures a + * temporary directory for out-of-order WAL data and registers an exit callback + * to clean up temporary files. + */ +void +init_archive_reader(XLogDumpPrivate *privateInfo, const char *waldir, + pg_compress_algorithm compression) +{ + int fd; + astreamer *streamer; + ArchivedWALEntry *entry = NULL; + XLogLongPageHeader longhdr; + + /* Open tar archive and store its file descriptor */ + fd = open_file_in_directory(waldir, privateInfo->archive_name); + + if (fd < 0) + pg_fatal("could not open file \"%s\"", privateInfo->archive_name); + + privateInfo->archive_fd = fd; + + streamer = astreamer_waldump_new(privateInfo); + + /* Before that we must parse the tar archive. */ + streamer = astreamer_tar_parser_new(streamer); + + /* Before that we must decompress, if archive is compressed. */ + if (compression == PG_COMPRESSION_GZIP) + streamer = astreamer_gzip_decompressor_new(streamer); + else if (compression == PG_COMPRESSION_LZ4) + streamer = astreamer_lz4_decompressor_new(streamer); + else if (compression == PG_COMPRESSION_ZSTD) + streamer = astreamer_zstd_decompressor_new(streamer); + + privateInfo->archive_streamer = streamer; + + /* Hash table storing WAL entries read from the archive */ + ArchivedWAL_HTAB = ArchivedWAL_create(16, NULL); + + /* + * Verify that the archive contains valid WAL files and fetch WAL segment + * size + */ + while (entry == NULL || entry->buf.len < XLOG_BLCKSZ) + { + if (read_archive_file(privateInfo, XLOG_BLCKSZ) == 0) + pg_fatal("could not find WAL in \"%s\" archive", + privateInfo->archive_name); + + entry = privateInfo->cur_wal; + } + + /* Set WalSegSz if WAL data is successfully read */ + longhdr = (XLogLongPageHeader) entry->buf.data; + + WalSegSz = longhdr->xlp_seg_size; + + if (!IsValidWalSegSize(WalSegSz)) + { + pg_log_error(ngettext("invalid WAL segment size in WAL file from archive \"%s\" (%d byte)", + "invalid WAL segment size in WAL file from archive \"%s\" (%d bytes)", + WalSegSz), + privateInfo->archive_name, WalSegSz); + pg_log_error_detail("The WAL segment size must be a power of two between 1 MB and 1 GB."); + exit(1); + } + + /* + * With the WAL segment size available, we can now initialize the + * dependent start and end segment numbers. + */ + Assert(!XLogRecPtrIsInvalid(privateInfo->startptr)); + XLByteToSeg(privateInfo->startptr, privateInfo->startSegNo, WalSegSz); + + if (XLogRecPtrIsInvalid(privateInfo->endptr)) + privateInfo->endSegNo = UINT64_MAX; + else + XLByteToSeg(privateInfo->endptr, privateInfo->endSegNo, WalSegSz); + + /* + * Setup temporary directory to store WAL segments and set up an exit + * callback to remove it upon completion. + */ + setup_tmpseg_dir(waldir); + atexit(cleanup_tmpseg_dir_atexit); +} + +/* + * Release the archive streamer chain and close the archive file. + */ +void +free_archive_reader(XLogDumpPrivate *privateInfo) +{ + /* + * NB: Normally, astreamer_finalize() is called before astreamer_free() to + * flush any remaining buffered data or to ensure the end of the tar + * archive is reached. However, when decoding a WAL file, once we hit the + * end LSN, any remaining WAL data in the buffer or the tar archive's + * unreached end can be safely ignored. + */ + astreamer_free(privateInfo->archive_streamer); + + /* Close the file. */ + if (close(privateInfo->archive_fd) != 0) + pg_log_error("could not close file \"%s\": %m", + privateInfo->archive_name); +} + +/* + * Copies WAL data from astreamer to readBuff; if unavailable, fetches more + * from the tar archive via astreamer. + */ +int +read_archive_wal_page(XLogDumpPrivate *privateInfo, XLogRecPtr targetPagePtr, + Size count, char *readBuff) +{ + char *p = readBuff; + Size nbytes = count; + XLogRecPtr recptr = targetPagePtr; + XLogSegNo segno; + ArchivedWALEntry *entry; + + XLByteToSeg(targetPagePtr, segno, WalSegSz); + entry = get_archive_wal_entry(segno, privateInfo); + + while (nbytes > 0) + { + char *buf = entry->buf.data; + int len = entry->buf.len; + + /* WAL record range that the buffer contains */ + XLogRecPtr endPtr; + XLogRecPtr startPtr; + + XLogSegNoOffsetToRecPtr(entry->segno, entry->total_read, + WalSegSz, endPtr); + startPtr = endPtr - len; + + /* + * pg_waldump may request to re-read the currently active page, but + * never a page older than the current one. Therefore, any fully + * consumed WAL data preceding the current page can be safely + * discarded. + */ + if (recptr >= endPtr) + { + /* Discard the buffered data */ + resetStringInfo(&entry->buf); + len = 0; + + /* + * Push back the partial page data for the current page to the + * buffer, ensuring it remains available for re-reading if + * requested. + */ + if (p > readBuff) + { + Assert((count - nbytes) > 0); + appendBinaryStringInfo(&entry->buf, readBuff, count - nbytes); + } + } + + if (len > 0 && recptr > startPtr) + { + int skipBytes = 0; + + /* + * The required offset is not at the start of the buffer, so skip + * bytes until reaching the desired offset of the target page. + */ + skipBytes = recptr - startPtr; + + buf += skipBytes; + len -= skipBytes; + } + + if (len > 0) + { + int readBytes = len >= nbytes ? nbytes : len; + + /* Ensure the reading page is in the buffer */ + Assert(recptr >= startPtr && recptr < endPtr); + + memcpy(p, buf, readBytes); + + /* Update state for read */ + nbytes -= readBytes; + p += readBytes; + recptr += readBytes; + } + else + { + /* + * Fetch more data; raise an error if it's not the current segment + * being read by the archive streamer or if reading of the + * archived file has finished. + */ + if (privateInfo->cur_wal != entry || + read_archive_file(privateInfo, READ_CHUNK_SIZE) == 0) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, privateInfo->timeline, entry->segno, + WalSegSz); + pg_fatal("could not read file \"%s\" from archive \"%s\": read %lld of %lld", + fname, privateInfo->archive_name, + (long long int) count - nbytes, + (long long int) nbytes); + } + } + } + + /* + * Should have either have successfully read all the requested bytes or + * reported a failure before this point. + */ + Assert(nbytes == 0); + + /* + * NB: We return the fixed value provided as input. Although we could + * return a boolean since we either successfully read the WAL page or + * raise an error, but the caller expects this value to be returned. The + * routine that reads WAL pages from the physical WAL file follows the + * same convention. + */ + return count; +} + +/* + * Reads the archive file and passes it to the archive streamer for + * decompression. + */ +static int +read_archive_file(XLogDumpPrivate *privateInfo, Size count) +{ + int rc; + char *buffer; + + buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8)); + + rc = read(privateInfo->archive_fd, buffer, count); + if (rc < 0) + pg_fatal("could not read file \"%s\": %m", + privateInfo->archive_name); + + /* + * Decompress (if required), and then parse the previously read contents + * of the tar file. + */ + if (rc > 0) + astreamer_content(privateInfo->archive_streamer, NULL, + buffer, rc, ASTREAMER_UNKNOWN); + pg_free(buffer); + + return rc; +} + +/* + * Returns the archived WAL entry from the hash table if it exists. Otherwise, + * it invokes the routine to read the archived file and retrieve the entry if + * it is not already present in the hash table. If the archive streamer happens + * to be reading a WAL from archive file that is not currently needed, that WAL + * data is written to a temporary file. + */ +static ArchivedWALEntry * +get_archive_wal_entry(XLogSegNo segno, XLogDumpPrivate *privateInfo) +{ + ArchivedWALEntry *entry = NULL; + char fname[MAXFNAMELEN]; + FILE *write_fp = NULL; + + /* Search hash table */ + entry = ArchivedWAL_lookup(ArchivedWAL_HTAB, segno); + + if (entry != NULL) + return entry; + + /* Needed WAL yet to be decoded from archive, do the same */ + while (1) + { + entry = privateInfo->cur_wal; + + /* Fetch more data */ + if (entry == NULL || entry->buf.len == 0) + { + if (read_archive_file(privateInfo, READ_CHUNK_SIZE) == 0) + break; /* archive file ended */ + } + + /* + * Either, here for the first time, or the archived streamer is + * reading a non-WAL file or an irrelevant WAL file. + */ + if (entry == NULL) + continue; + + /* Found the required entry */ + if (entry->segno == segno) + return entry; + + /* + * Ignore if the timeline is different or the current segment is not + * the desired one. + */ + if (privateInfo->timeline != entry->timeline || + privateInfo->startSegNo > entry->segno || + privateInfo->endSegNo < entry->segno) + { + privateInfo->cur_wal = NULL; + continue; + } + + /* + * Archive streamer is currently reading a file that isn't the one + * asked for, but it's required for a future feature. It should be + * written to a temporary location for retrieval when needed. + */ + + /* Create a temporary file if one does not already exist */ + if (!entry->tmpseg_exists) + { + write_fp = prepare_tmp_write(entry->segno); + entry->tmpseg_exists = true; + } + + /* Flush data from the buffer to the file */ + perform_tmp_write(entry->segno, &entry->buf, write_fp); + resetStringInfo(&entry->buf); + + /* + * The change in the current segment entry indicates that the reading + * of this file has ended. + */ + if (entry != privateInfo->cur_wal && write_fp != NULL) + { + fclose(write_fp); + write_fp = NULL; + } + } + + /* Requested WAL segment not found */ + XLogFileName(fname, privateInfo->timeline, segno, WalSegSz); + pg_fatal("could not find file \"%s\" in archive", fname); +} + +/* + * Set up a temporary directory to temporarily store WAL segments. + */ +static void +setup_tmpseg_dir(const char *waldir) +{ + char *template; + + /* + * Use the directory specified by the TMPDIR environment variable. If it's + * not set, use the provided WAL directory to extract WAL file + * temporarily. + */ + template = psprintf("%s/waldump_tmp-XXXXXX", + getenv("TMPDIR") ? getenv("TMPDIR") : waldir); + TmpWalSegDir = mkdtemp(template); + + if (TmpWalSegDir == NULL) + pg_fatal("could not create directory \"%s\": %m", template); + + canonicalize_path(TmpWalSegDir); + + pg_log_debug("created directory \"%s\"", TmpWalSegDir); +} + +/* + * Removes the temporarily store WAL segments, if any, at exiting. + */ +static void +cleanup_tmpseg_dir_atexit(void) +{ + ArchivedWAL_iterator it; + ArchivedWALEntry *entry; + + /* Remove temporary segments */ + ArchivedWAL_start_iterate(ArchivedWAL_HTAB, &it); + while ((entry = ArchivedWAL_iterate(ArchivedWAL_HTAB, &it)) != NULL) + { + if (entry->tmpseg_exists) + { + remove_tmp_walseg(entry->segno, false); + entry->tmpseg_exists = false; + } + } + + /* Remove temporary directory */ + if (rmdir(TmpWalSegDir) == 0) + pg_log_debug("removed directory \"%s\"", TmpWalSegDir); +} + +/* + * Generate the temporary WAL file path. + * + * Note that the caller is responsible to pfree it. + */ +char * +get_tmp_walseg_path(XLogSegNo segno) +{ + char *fpath = (char *) palloc(MAXPGPATH); + + Assert(TmpWalSegDir); + + snprintf(fpath, MAXPGPATH, "%s/%08X%08X", + TmpWalSegDir, + (uint32) (segno / XLogSegmentsPerXLogId(WalSegSz)), + (uint32) (segno % XLogSegmentsPerXLogId(WalSegSz))); + + return fpath; +} + +/* + * Routine to check whether a temporary file exists for the corresponding WAL + * segment number. + */ +bool +tmp_walseg_exists(XLogSegNo segno) +{ + ArchivedWALEntry *entry; + + entry = ArchivedWAL_lookup(ArchivedWAL_HTAB, segno); + + if (entry == NULL) + return false; + + return entry->tmpseg_exists; +} + +/* + * Create an empty placeholder file and return its handle. + */ +static FILE * +prepare_tmp_write(XLogSegNo segno) +{ + FILE *file; + char *fpath; + + fpath = get_tmp_walseg_path(segno); + + /* Create an empty placeholder */ + file = fopen(fpath, PG_BINARY_W); + if (file == NULL) + pg_fatal("could not create file \"%s\": %m", fpath); + +#ifndef WIN32 + if (chmod(fpath, pg_file_create_mode)) + pg_fatal("could not set permissions on file \"%s\": %m", + fpath); +#endif + + pg_log_debug("temporarily exporting file \"%s\"", fpath); + pfree(fpath); + + return file; +} + +/* + * Write buffer data to the given file handle. + */ +static void +perform_tmp_write(XLogSegNo segno, StringInfo buf, FILE *file) +{ + Assert(file); + + errno = 0; + if (buf->len > 0 && fwrite(buf->data, buf->len, 1, file) != 1) + { + /* + * If write didn't set errno, assume problem is no disk space + */ + if (errno == 0) + errno = ENOSPC; + pg_fatal("could not write to file \"%s\": %m", + get_tmp_walseg_path(segno)); + } +} + +/* + * Remove temporary file + */ +void +remove_tmp_walseg(XLogSegNo segno, bool update_entry) +{ + char *fpath = get_tmp_walseg_path(segno); + + if (unlink(fpath) == 0) + pg_log_debug("removed file \"%s\"", fpath); + pfree(fpath); + + /* Update entry if requested */ + if (update_entry) + { + ArchivedWALEntry *entry; + + entry = ArchivedWAL_lookup(ArchivedWAL_HTAB, segno); + Assert(entry != NULL); + entry->tmpseg_exists = false; + } +} + +/* + * Create an astreamer that can read WAL from tar file. + */ +static astreamer * +astreamer_waldump_new(XLogDumpPrivate *privateInfo) +{ + astreamer_waldump *streamer; + + streamer = palloc0(sizeof(astreamer_waldump)); + *((const astreamer_ops **) &streamer->base.bbs_ops) = + &astreamer_waldump_ops; + + streamer->privateInfo = privateInfo; + + return &streamer->base; +} + +/* + * Main entry point of the archive streamer for reading WAL data from a tar + * file. If a member is identified as a valid WAL file, a hash entry is created + * for it, and its contents are copied into that entry's buffer, making them + * accessible to the decoding routine. + */ +static void +astreamer_waldump_content(astreamer *streamer, astreamer_member *member, + const char *data, int len, + astreamer_archive_context context) +{ + astreamer_waldump *mystreamer = (astreamer_waldump *) streamer; + XLogDumpPrivate *privateInfo = mystreamer->privateInfo; + + Assert(context != ASTREAMER_UNKNOWN); + + switch (context) + { + case ASTREAMER_MEMBER_HEADER: + { + XLogSegNo segno; + TimeLineID timeline; + ArchivedWALEntry *entry; + bool found; + + pg_log_debug("reading \"%s\"", member->pathname); + + if (!member_is_wal_file(mystreamer, member, + &segno, &timeline)) + break; + + entry = ArchivedWAL_insert(ArchivedWAL_HTAB, segno, &found); + + /* + * Shouldn't happen, but if it does, simply ignore the + * duplicate WAL file. + */ + if (found) + { + pg_log_warning("ignoring duplicate WAL file found in archive: \"%s\"", + member->pathname); + break; + } + + initStringInfo(&entry->buf); + entry->timeline = timeline; + entry->total_read = 0; + + privateInfo->cur_wal = entry; + } + break; + + case ASTREAMER_MEMBER_CONTENTS: + if (privateInfo->cur_wal) + { + appendBinaryStringInfo(&privateInfo->cur_wal->buf, data, len); + privateInfo->cur_wal->total_read += len; + } + break; + + case ASTREAMER_MEMBER_TRAILER: + privateInfo->cur_wal = NULL; + break; + + case ASTREAMER_ARCHIVE_TRAILER: + break; + + default: + /* Shouldn't happen. */ + pg_fatal("unexpected state while parsing tar file"); + } +} + +/* + * End-of-stream processing for an astreamer_waldump stream. + */ +static void +astreamer_waldump_finalize(astreamer *streamer) +{ + Assert(streamer->bbs_next == NULL); +} + +/* + * Free memory associated with a astreamer_waldump stream. + */ +static void +astreamer_waldump_free(astreamer *streamer) +{ + Assert(streamer->bbs_next == NULL); + pfree(streamer); +} + +/* + * Returns true if the archive member name matches the WAL naming format. If + * successful, it also outputs the WAL segment number, and timeline. + */ +static bool +member_is_wal_file(astreamer_waldump *mystreamer, astreamer_member *member, + XLogSegNo *curSegNo, TimeLineID *curTimeline) +{ + int pathlen; + XLogSegNo segNo; + TimeLineID timeline; + char *fname; + + /* We are only interested in normal files. */ + if (member->is_directory || member->is_link) + return false; + + pathlen = strlen(member->pathname); + if (pathlen < XLOG_FNAME_LEN) + return false; + + /* WAL file could be with full path */ + fname = member->pathname + (pathlen - XLOG_FNAME_LEN); + if (!IsXLogFileName(fname)) + return false; + + /* + * XXX: On some systems (e.g., OpenBSD), the tar utility includes + * PaxHeaders when creating an archive. These are special entries that + * store extended metadata for the file entry immediately following them, + * and they share the exact same name as that file. + */ + if (strstr(member->pathname, "PaxHeaders.")) + return false; + + /* Parse position from file */ + XLogFromFileName(fname, &timeline, &segNo, WalSegSz); + + *curSegNo = segNo; + *curTimeline = timeline; + + return true; +} diff --git a/src/bin/pg_waldump/meson.build b/src/bin/pg_waldump/meson.build index 937e0d688414..f31e0d1cd86f 100644 --- a/src/bin/pg_waldump/meson.build +++ b/src/bin/pg_waldump/meson.build @@ -1,6 +1,7 @@ # Copyright (c) 2022-2025, PostgreSQL Global Development Group pg_waldump_sources = files( + 'archive_waldump.c', 'compat.c', 'pg_waldump.c', 'rmgrdesc.c', @@ -18,7 +19,7 @@ endif pg_waldump = executable('pg_waldump', pg_waldump_sources, - dependencies: [frontend_code, lz4, zstd], + dependencies: [frontend_code, lz4, zstd, libpq], c_args: ['-DFRONTEND'], # needed for xlogreader et al kwargs: default_bin_args, ) @@ -29,6 +30,7 @@ tests += { 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), 'tap': { + 'env': {'TAR': tar.found() ? tar.full_path() : ''}, 'tests': [ 't/001_basic.pl', 't/002_save_fullpage.pl', diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 1c1ccf59f653..034f70d1410b 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -29,6 +29,7 @@ #include "common/logging.h" #include "common/relpath.h" #include "getopt_long.h" +#include "pg_waldump.h" #include "rmgrdesc.h" #include "storage/bufpage.h" @@ -37,21 +38,14 @@ * give a thought about doing the same in pg_walinspect contrib module as well. */ +int WalSegSz = DEFAULT_XLOG_SEG_SIZE; + static const char *progname; -static int WalSegSz; static volatile sig_atomic_t time_to_stop = false; static const RelFileLocator emptyRelFileLocator = {0, 0, 0}; -typedef struct XLogDumpPrivate -{ - TimeLineID timeline; - XLogRecPtr startptr; - XLogRecPtr endptr; - bool endptr_reached; -} XLogDumpPrivate; - typedef struct XLogDumpConfig { /* display options */ @@ -184,7 +178,7 @@ split_path(const char *path, char **dir, char **fname) * * return a read only fd */ -static int +int open_file_in_directory(const char *directory, const char *fname) { int fd = -1; @@ -333,6 +327,35 @@ identify_target_directory(char *directory, char *fname) return NULL; /* not reached */ } +/* + * Returns the size in bytes of the data to be read. Returns -1 if the end + * point has already been reached. + */ +static inline int +required_read_len(XLogDumpPrivate *private, XLogRecPtr targetPagePtr, + int reqLen) +{ + int count = XLOG_BLCKSZ; + + if (unlikely(private->endptr_reached)) + return -1; + + if (XLogRecPtrIsValid(private->endptr)) + { + if (targetPagePtr + XLOG_BLCKSZ <= private->endptr) + count = XLOG_BLCKSZ; + else if (targetPagePtr + reqLen <= private->endptr) + count = private->endptr - targetPagePtr; + else + { + private->endptr_reached = true; + return -1; + } + } + + return count; +} + /* pg_waldump's XLogReaderRoutine->segment_open callback */ static void WALDumpOpenSegment(XLogReaderState *state, XLogSegNo nextSegNo, @@ -390,21 +413,12 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetPtr, char *readBuff) { XLogDumpPrivate *private = state->private_data; - int count = XLOG_BLCKSZ; + int count = required_read_len(private, targetPagePtr, reqLen); WALReadError errinfo; - if (XLogRecPtrIsValid(private->endptr)) - { - if (targetPagePtr + XLOG_BLCKSZ <= private->endptr) - count = XLOG_BLCKSZ; - else if (targetPagePtr + reqLen <= private->endptr) - count = private->endptr - targetPagePtr; - else - { - private->endptr_reached = true; - return -1; - } - } + /* Bail out if the count to be read is not valid */ + if (count < 0) + return -1; if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline, &errinfo)) @@ -430,6 +444,84 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, return count; } +/* + * pg_waldump's XLogReaderRoutine->segment_open callback to support dumping WAL + * files from tar archives. + */ +static void +TarWALDumpOpenSegment(XLogReaderState *state, XLogSegNo nextSegNo, + TimeLineID *tli_p) +{ + /* No action needed */ +} + +/* + * pg_waldump's XLogReaderRoutine->segment_close callback. + */ +static void +TarWALDumpCloseSegment(XLogReaderState *state) +{ + /* No action needed */ +} + +/* + * pg_waldump's XLogReaderRoutine->page_read callback to support dumping WAL + * files from tar archives. + */ +static int +TarWALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, + XLogRecPtr targetPtr, char *readBuff) +{ + XLogDumpPrivate *private = state->private_data; + int count = required_read_len(private, targetPagePtr, reqLen); + XLogSegNo nextSegNo; + + /* Bail out if the count to be read is not valid */ + if (count < 0) + return -1; + + /* + * If the target page is in a different segment, first check for the WAL + * segment's physical existence in the temporary directory. + */ + nextSegNo = state->seg.ws_segno; + if (!XLByteInSeg(targetPagePtr, nextSegNo, WalSegSz)) + { + if (state->seg.ws_file >= 0) + { + close(state->seg.ws_file); + state->seg.ws_file = -1; + + /* Remove this file, as it is no longer needed. */ + remove_tmp_walseg(nextSegNo, true); + } + + XLByteToSeg(targetPagePtr, nextSegNo, WalSegSz); + state->seg.ws_tli = private->timeline; + state->seg.ws_segno = nextSegNo; + + /* + * If the next segment exists, open it and continue reading from there + */ + if (tmp_walseg_exists(nextSegNo)) + { + char *fpath; + + fpath = get_tmp_walseg_path(nextSegNo); + state->seg.ws_file = open(fpath, O_RDONLY | PG_BINARY, 0); + pfree(fpath); + } + } + + /* Continue reading from the open WAL segment, if any */ + if (state->seg.ws_file >= 0) + return WALDumpReadPage(state, targetPagePtr, count, targetPtr, + readBuff); + + /* Otherwise, read the WAL page from the archive streamer */ + return read_archive_wal_page(private, targetPagePtr, count, readBuff); +} + /* * Boolean to return whether the given WAL record matches a specific relation * and optionally block. @@ -767,8 +859,8 @@ usage(void) printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n" " valid names are main, fsm, vm, init\n")); printf(_(" -n, --limit=N number of records to display\n")); - printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n" - " directory with a ./pg_wal that contains such files\n" + printf(_(" -p, --path=PATH tar archive or a directory in which to find WAL segment files or\n" + " a directory with a ./pg_wal that contains such files\n" " (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n")); printf(_(" -q, --quiet do not print any output, except for errors\n")); printf(_(" -r, --rmgr=RMGR only show records generated by resource manager RMGR;\n" @@ -800,7 +892,10 @@ main(int argc, char **argv) XLogRecord *record; XLogRecPtr first_record; char *waldir = NULL; + char *walpath = NULL; char *errormsg; + bool is_archive = false; + pg_compress_algorithm compression; static struct option long_options[] = { {"bkp-details", no_argument, NULL, 'b'}, @@ -932,7 +1027,7 @@ main(int argc, char **argv) } break; case 'p': - waldir = pg_strdup(optarg); + walpath = pg_strdup(optarg); break; case 'q': config.quiet = true; @@ -1096,10 +1191,20 @@ main(int argc, char **argv) goto bad_argument; } - if (waldir != NULL) + if (walpath != NULL) { + /* validate path points to tar archive */ + if (is_archive_file(walpath, &compression)) + { + char *fname = NULL; + + split_path(walpath, &waldir, &fname); + + private.archive_name = fname; + is_archive = true; + } /* validate path points to directory */ - if (!verify_directory(waldir)) + else if (!verify_directory(walpath)) { pg_log_error("could not open directory \"%s\": %m", waldir); goto bad_argument; @@ -1117,6 +1222,17 @@ main(int argc, char **argv) int fd; XLogSegNo segno; + /* + * If a tar archive is passed using the --path option, all other + * arguments become unnecessary. + */ + if (is_archive) + { + pg_log_error("unnecessary command-line arguments specified with tar archive (first is \"%s\")", + argv[optind]); + goto bad_argument; + } + split_path(argv[optind], &directory, &fname); if (waldir == NULL && directory != NULL) @@ -1127,69 +1243,77 @@ main(int argc, char **argv) pg_fatal("could not open directory \"%s\": %m", waldir); } - waldir = identify_target_directory(waldir, fname); - fd = open_file_in_directory(waldir, fname); - if (fd < 0) - pg_fatal("could not open file \"%s\"", fname); - close(fd); - - /* parse position from file */ - XLogFromFileName(fname, &private.timeline, &segno, WalSegSz); - - if (!XLogRecPtrIsValid(private.startptr)) - XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, private.startptr); - else if (!XLByteInSeg(private.startptr, segno, WalSegSz)) + if (fname != NULL && is_archive_file(fname, &compression)) { - pg_log_error("start WAL location %X/%08X is not inside file \"%s\"", - LSN_FORMAT_ARGS(private.startptr), - fname); - goto bad_argument; + private.archive_name = fname; + is_archive = true; } - - /* no second file specified, set end position */ - if (!(optind + 1 < argc) && !XLogRecPtrIsValid(private.endptr)) - XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, private.endptr); - - /* parse ENDSEG if passed */ - if (optind + 1 < argc) + else { - XLogSegNo endsegno; - - /* ignore directory, already have that */ - split_path(argv[optind + 1], &directory, &fname); - + waldir = identify_target_directory(waldir, fname); fd = open_file_in_directory(waldir, fname); if (fd < 0) pg_fatal("could not open file \"%s\"", fname); close(fd); /* parse position from file */ - XLogFromFileName(fname, &private.timeline, &endsegno, WalSegSz); + XLogFromFileName(fname, &private.timeline, &segno, WalSegSz); - if (endsegno < segno) - pg_fatal("ENDSEG %s is before STARTSEG %s", - argv[optind + 1], argv[optind]); + if (!XLogRecPtrIsValid(private.startptr)) + XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, private.startptr); + else if (!XLByteInSeg(private.startptr, segno, WalSegSz)) + { + pg_log_error("start WAL location %X/%08X is not inside file \"%s\"", + LSN_FORMAT_ARGS(private.startptr), + fname); + goto bad_argument; + } - if (!XLogRecPtrIsValid(private.endptr)) - XLogSegNoOffsetToRecPtr(endsegno + 1, 0, WalSegSz, - private.endptr); + /* no second file specified, set end position */ + if (!(optind + 1 < argc) && !XLogRecPtrIsValid(private.endptr)) + XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, private.endptr); - /* set segno to endsegno for check of --end */ - segno = endsegno; - } + /* parse ENDSEG if passed */ + if (optind + 1 < argc) + { + XLogSegNo endsegno; + /* ignore directory, already have that */ + split_path(argv[optind + 1], &directory, &fname); - if (!XLByteInSeg(private.endptr, segno, WalSegSz) && - private.endptr != (segno + 1) * WalSegSz) - { - pg_log_error("end WAL location %X/%08X is not inside file \"%s\"", - LSN_FORMAT_ARGS(private.endptr), - argv[argc - 1]); - goto bad_argument; + fd = open_file_in_directory(waldir, fname); + if (fd < 0) + pg_fatal("could not open file \"%s\"", fname); + close(fd); + + /* parse position from file */ + XLogFromFileName(fname, &private.timeline, &endsegno, WalSegSz); + + if (endsegno < segno) + pg_fatal("ENDSEG %s is before STARTSEG %s", + argv[optind + 1], argv[optind]); + + if (!XLogRecPtrIsValid(private.endptr)) + XLogSegNoOffsetToRecPtr(endsegno + 1, 0, WalSegSz, + private.endptr); + + /* set segno to endsegno for check of --end */ + segno = endsegno; + } + + + if (!XLByteInSeg(private.endptr, segno, WalSegSz) && + private.endptr != (segno + 1) * WalSegSz) + { + pg_log_error("end WAL location %X/%08X is not inside file \"%s\"", + LSN_FORMAT_ARGS(private.endptr), + argv[argc - 1]); + goto bad_argument; + } } } - else - waldir = identify_target_directory(waldir, NULL); + else if (!is_archive) + waldir = identify_target_directory(walpath, NULL); /* we don't know what to print */ if (!XLogRecPtrIsValid(private.startptr)) @@ -1201,12 +1325,36 @@ main(int argc, char **argv) /* done with argument parsing, do the actual work */ /* we have everything we need, start reading */ - xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, - XL_ROUTINE(.page_read = WALDumpReadPage, - .segment_open = WALDumpOpenSegment, - .segment_close = WALDumpCloseSegment), - &private); + if (is_archive) + { + /* + * A NULL WAL directory indicates that the archive file is located in + * the current working directory of the pg_waldump execution + */ + waldir = waldir ? pg_strdup(waldir) : pg_strdup("."); + + /* Set up for reading tar file */ + init_archive_reader(&private, waldir, compression); + + /* Routine to decode WAL files in tar archive */ + xlogreader_state = + XLogReaderAllocate(WalSegSz, waldir, + XL_ROUTINE(.page_read = TarWALDumpReadPage, + .segment_open = TarWALDumpOpenSegment, + .segment_close = TarWALDumpCloseSegment), + &private); + } + else + { + /* Routine to decode WAL files */ + xlogreader_state = + XLogReaderAllocate(WalSegSz, waldir, + XL_ROUTINE(.page_read = WALDumpReadPage, + .segment_open = WALDumpOpenSegment, + .segment_close = WALDumpCloseSegment), + &private); + } + if (!xlogreader_state) pg_fatal("out of memory while allocating a WAL reading processor"); @@ -1315,6 +1463,9 @@ main(int argc, char **argv) XLogReaderFree(xlogreader_state); + if (is_archive) + free_archive_reader(&private); + return EXIT_SUCCESS; bad_argument: diff --git a/src/bin/pg_waldump/pg_waldump.h b/src/bin/pg_waldump/pg_waldump.h new file mode 100644 index 000000000000..03e02625ba1c --- /dev/null +++ b/src/bin/pg_waldump/pg_waldump.h @@ -0,0 +1,65 @@ +/*------------------------------------------------------------------------- + * + * pg_waldump.h - decode and display WAL + * + * Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_waldump/pg_waldump.h + *------------------------------------------------------------------------- + */ +#ifndef PG_WALDUMP_H +#define PG_WALDUMP_H + +#include "access/xlogdefs.h" +#include "fe_utils/astreamer.h" + +extern int WalSegSz; + +/* Forward declaration */ +struct ArchivedWALEntry; + +/* Contains the necessary information to drive WAL decoding */ +typedef struct XLogDumpPrivate +{ + TimeLineID timeline; + XLogRecPtr startptr; + XLogRecPtr endptr; + bool endptr_reached; + + /* Fields required to read WAL from archive */ + char *archive_name; /* Tar archive name */ + int archive_fd; /* File descriptor for the open tar file */ + + astreamer *archive_streamer; + + /* What the archive streamer is currently reading */ + struct ArchivedWALEntry *cur_wal; + + /* + * Although these values can be easily derived from startptr and endptr, + * doing so repeatedly for each archived member would be inefficient, as + * it would involve recalculating and filtering out irrelevant WAL + * segments. + */ + XLogSegNo startSegNo; + XLogSegNo endSegNo; +} XLogDumpPrivate; + +extern int open_file_in_directory(const char *directory, const char *fname); + +extern bool is_archive_file(const char *fname, + pg_compress_algorithm *compression); +extern void init_archive_reader(XLogDumpPrivate *privateInfo, + const char *waldir, + pg_compress_algorithm compression); +extern void free_archive_reader(XLogDumpPrivate *privateInfo); +extern int read_archive_wal_page(XLogDumpPrivate *privateInfo, + XLogRecPtr targetPagePtr, + Size count, char *readBuff); + +extern char *get_tmp_walseg_path(XLogSegNo segno); +extern bool tmp_walseg_exists(XLogSegNo segno); +extern void remove_tmp_walseg(XLogSegNo segno, bool update_entry); + +#endif /* end of PG_WALDUMP_H */ diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index f26d75e01cfd..d752b5dd6567 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -3,9 +3,13 @@ use strict; use warnings FATAL => 'all'; +use Cwd; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +use List::Util qw(shuffle); + +my $tar = $ENV{TAR}; program_help_ok('pg_waldump'); program_version_ok('pg_waldump'); @@ -198,28 +202,6 @@ ], qr/./, 'runs with start and end segment specified'); -command_fails_like( - [ 'pg_waldump', '--path' => $node->data_dir ], - qr/error: no start WAL location given/, - 'path option requires start location'); -command_like( - [ - 'pg_waldump', - '--path' => $node->data_dir, - '--start' => $start_lsn, - '--end' => $end_lsn, - ], - qr/./, - 'runs with path option and start and end locations'); -command_fails_like( - [ - 'pg_waldump', - '--path' => $node->data_dir, - '--start' => $start_lsn, - ], - qr/error: error in WAL record at/, - 'falling off the end of the WAL results in an error'); - command_like( [ 'pg_waldump', '--quiet', @@ -227,15 +209,6 @@ ], qr/^$/, 'no output with --quiet option'); -command_fails_like( - [ - 'pg_waldump', '--quiet', - '--path' => $node->data_dir, - '--start' => $start_lsn - ], - qr/error: error in WAL record at/, - 'errors are shown with --quiet'); - # Test for: Display a message that we're skipping data if `from` # wasn't a pointer to the start of a record. @@ -266,15 +239,15 @@ sub test_pg_waldump { local $Test::Builder::Level = $Test::Builder::Level + 1; - my @opts = @_; + my ($path, @opts) = @_; my ($stdout, $stderr); my $result = IPC::Run::run [ 'pg_waldump', - '--path' => $node->data_dir, '--start' => $start_lsn, '--end' => $end_lsn, + '--path' => $path, @opts ], '>' => \$stdout, @@ -286,40 +259,139 @@ sub test_pg_waldump return @lines; } -my @lines; - -@lines = test_pg_waldump; -is(grep(!/^rmgr: \w/, @lines), 0, 'all output lines are rmgr lines'); - -@lines = test_pg_waldump('--limit' => 6); -is(@lines, 6, 'limit option observed'); - -@lines = test_pg_waldump('--fullpage'); -is(grep(!/^rmgr:.*\bFPW\b/, @lines), 0, 'all output lines are FPW'); - -@lines = test_pg_waldump('--stats'); -like($lines[0], qr/WAL statistics/, "statistics on stdout"); -is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); - -@lines = test_pg_waldump('--stats=record'); -like($lines[0], qr/WAL statistics/, "statistics on stdout"); -is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); - -@lines = test_pg_waldump('--rmgr' => 'Btree'); -is(grep(!/^rmgr: Btree/, @lines), 0, 'only Btree lines'); - -@lines = test_pg_waldump('--fork' => 'init'); -is(grep(!/fork init/, @lines), 0, 'only init fork lines'); - -@lines = test_pg_waldump( - '--relation' => "$default_ts_oid/$postgres_db_oid/$rel_t1_oid"); -is(grep(!/rel $default_ts_oid\/$postgres_db_oid\/$rel_t1_oid/, @lines), - 0, 'only lines for selected relation'); - -@lines = test_pg_waldump( - '--relation' => "$default_ts_oid/$postgres_db_oid/$rel_i1a_oid", - '--block' => 1); -is(grep(!/\bblk 1\b/, @lines), 0, 'only lines for selected block'); +# Create a tar archive, sorting the file order +sub generate_archive +{ + my ($archive, $directory, $compression_flags) = @_; + + my @files; + opendir my $dh, $directory or die "opendir: $!"; + while (my $entry = readdir $dh) { + # Skip '.' and '..' + next if $entry eq '.' || $entry eq '..'; + push @files, $entry; + } + closedir $dh; + + @files = shuffle @files; + + # move into the WAL directory before archiving files + my $cwd = getcwd; + chdir($directory) || die "chdir: $!"; + command_ok([$tar, $compression_flags, $archive, @files]); + chdir($cwd) || die "chdir: $!"; +} +my $tmp_dir = PostgreSQL::Test::Utils::tempdir_short(); + +my @scenarios = ( + { + 'path' => $node->data_dir, + 'is_archive' => 0, + 'enabled' => 1 + }, + { + 'path' => "$tmp_dir/pg_wal.tar", + 'compression_method' => 'none', + 'compression_flags' => '-cf', + 'is_archive' => 1, + 'enabled' => 1 + }, + { + 'path' => "$tmp_dir/pg_wal.tar.gz", + 'compression_method' => 'gzip', + 'compression_flags' => '-czf', + 'is_archive' => 1, + 'enabled' => check_pg_config("#define HAVE_LIBZ 1") + }); + +for my $scenario (@scenarios) +{ + my $path = $scenario->{'path'}; + + SKIP: + { + skip "tar command is not available", 3 + if !defined $tar; + skip "$scenario->{'compression_method'} compression not supported by this build", 3 + if !$scenario->{'enabled'} && $scenario->{'is_archive'}; + + # create pg_wal archive + if ($scenario->{'is_archive'}) + { + generate_archive($path, + $node->data_dir . '/pg_wal', + $scenario->{'compression_flags'}); + } + + command_fails_like( + [ 'pg_waldump', '--path' => $path ], + qr/error: no start WAL location given/, + 'path option requires start location'); + command_like( + [ + 'pg_waldump', + '--path' => $path, + '--start' => $start_lsn, + '--end' => $end_lsn, + ], + qr/./, + 'runs with path option and start and end locations'); + command_fails_like( + [ + 'pg_waldump', + '--path' => $path, + '--start' => $start_lsn, + ], + qr/error: error in WAL record at/, + 'falling off the end of the WAL results in an error'); + + command_fails_like( + [ + 'pg_waldump', '--quiet', + '--path' => $path, + '--start' => $start_lsn + ], + qr/error: error in WAL record at/, + 'errors are shown with --quiet'); + + my @lines; + @lines = test_pg_waldump($path); + is(grep(!/^rmgr: \w/, @lines), 0, 'all output lines are rmgr lines'); + + @lines = test_pg_waldump($path, '--limit' => 6); + is(@lines, 6, 'limit option observed'); + + @lines = test_pg_waldump($path, '--fullpage'); + is(grep(!/^rmgr:.*\bFPW\b/, @lines), 0, 'all output lines are FPW'); + + @lines = test_pg_waldump($path, '--stats'); + like($lines[0], qr/WAL statistics/, "statistics on stdout"); + is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); + + @lines = test_pg_waldump($path, '--stats=record'); + like($lines[0], qr/WAL statistics/, "statistics on stdout"); + is(grep(/^rmgr:/, @lines), 0, 'no rmgr lines output'); + + @lines = test_pg_waldump($path, '--rmgr' => 'Btree'); + is(grep(!/^rmgr: Btree/, @lines), 0, 'only Btree lines'); + + @lines = test_pg_waldump($path, '--fork' => 'init'); + is(grep(!/fork init/, @lines), 0, 'only init fork lines'); + + @lines = test_pg_waldump($path, + '--relation' => "$default_ts_oid/$postgres_db_oid/$rel_t1_oid"); + is(grep(!/rel $default_ts_oid\/$postgres_db_oid\/$rel_t1_oid/, @lines), + 0, 'only lines for selected relation'); + + @lines = test_pg_waldump($path, + '--relation' => "$default_ts_oid/$postgres_db_oid/$rel_i1a_oid", + '--block' => 1); + is(grep(!/\bblk 1\b/, @lines), 0, 'only lines for selected block'); + + # Cleanup. + unlink $path if $scenario->{'is_archive'}; + } +} done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 3451538565e8..493d889016a8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -144,6 +144,8 @@ ArchiveOpts ArchiveShutdownCB ArchiveStartupCB ArchiveStreamState +ArchivedWALEntry +ArchivedWAL_hash ArchiverOutput ArchiverStage ArrayAnalyzeExtraData @@ -3494,6 +3496,7 @@ astreamer_recovery_injector astreamer_tar_archiver astreamer_tar_parser astreamer_verify +astreamer_waldump astreamer_zstd_frame auth_password_hook_typ autovac_table