Fix various checksum check problems for pg_verify_checksums and base backups

Three issues are fixed in this patch:
- Base backups forgot to ignore files specific to EXEC_BACKEND, leading
to spurious warnings when checksums are enabled, per analysis from me.
- pg_verify_checksums forgot about files specific to EXEC_BACKEND,
leading to failures of the tool on any such build, particularly Windows.
This error was originally found by newly-introduced TAP tests in various
buildfarm members using EXEC_BACKEND.
- pg_verify_checksums forgot to count for temporary files and temporary
paths, which could be valid relation files, without checksums, per
report from Andres Freund.  More tests are added to cover this case.

A new test case which emulates corruption for a file in a different
tablespace is added, coming from from Michael Banck, while I have coded
the main code and refactored the test code.

Author: Michael Banck, Michael Paquier
Reviewed-by: Stephen Frost, David Steele
Discussion: https://postgr.es/m/20181021134206.GA14282@paquier.xyz

diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 3f1eae38a92fca5240edd72bea43dc5ff022da8f..15333259cc70d330f91a3eb2a0e87ef9e656b8e5 100644 (file)
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -189,12 +189,19 @@ static const char *excludeFiles[] =
 
 /*
  * List of files excluded from checksum validation.
+ *
+ * Note: this list should be kept in sync with what pg_verify_checksums.c
+ * includes.
  */
 static const char *noChecksumFiles[] = {
    "pg_control",
    "pg_filenode.map",
    "pg_internal.init",
    "PG_VERSION",
+#ifdef EXEC_BACKEND
+   "config_exec_params",
+   "config_exec_params.new",
+#endif
    NULL,
 };
 

diff --git a/src/bin/pg_verify_checksums/pg_verify_checksums.c b/src/bin/pg_verify_checksums/pg_verify_checksums.c
index 589a3cc58988f2716041efa5519a608196824e4f..3ad69451ab693b68bfacedee0c36132a46ad0eb6 100644 (file)
--- a/src/bin/pg_verify_checksums/pg_verify_checksums.c
+++ b/src/bin/pg_verify_checksums/pg_verify_checksums.c
@@ -20,6 +20,7 @@
 #include "storage/bufpage.h"
 #include "storage/checksum.h"
 #include "storage/checksum_impl.h"
+#include "storage/fd.h"
 
 
 static int64 files = 0;
@@ -49,11 +50,20 @@ usage(void)
    printf(_("Report bugs to <pgsql-bugs@postgresql.org>.\n"));
 }
 
+/*
+ * List of files excluded from checksum validation.
+ *
+ * Note: this list should be kept in sync with what basebackup.c includes.
+ */
 static const char *const skip[] = {
    "pg_control",
    "pg_filenode.map",
    "pg_internal.init",
    "PG_VERSION",
+#ifdef EXEC_BACKEND
+   "config_exec_params",
+   "config_exec_params.new",
+#endif
    NULL,
 };
 
@@ -62,13 +72,10 @@ skipfile(const char *fn)
 {
    const char *const *f;
 
-   if (strcmp(fn, ".") == 0 ||
-       strcmp(fn, "..") == 0)
-       return true;
-
    for (f = skip; *f; f++)
        if (strcmp(*f, fn) == 0)
            return true;
+
    return false;
 }
 
@@ -146,9 +153,22 @@ scan_directory(const char *basedir, const char *subdir)
        char        fn[MAXPGPATH];
        struct stat st;
 
-       if (skipfile(de->d_name))
+       if (strcmp(de->d_name, ".") == 0 ||
+           strcmp(de->d_name, "..") == 0)
            continue;
 
+       /* Skip temporary files */
+       if (strncmp(de->d_name,
+                   PG_TEMP_FILE_PREFIX,
+                   strlen(PG_TEMP_FILE_PREFIX)) == 0)
+           continue;
+
+       /* Skip temporary folders */
+       if (strncmp(de->d_name,
+                   PG_TEMP_FILES_DIR,
+                   strlen(PG_TEMP_FILES_DIR)) == 0)
+           return;
+
        snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name);
        if (lstat(fn, &st) < 0)
        {
@@ -163,6 +183,9 @@ scan_directory(const char *basedir, const char *subdir)
                       *segmentpath;
            BlockNumber segmentno = 0;
 
+           if (skipfile(de->d_name))
+               continue;
+
            /*
             * Cut off at the segment boundary (".") to get the segment number
             * in order to mix it into the checksum. Then also cut off at the