diff options
Diffstat (limited to 'src/backend')
| -rw-r--r-- | src/backend/access/rmgrdesc/xlogdesc.c | 16 | ||||
| -rw-r--r-- | src/backend/access/transam/xlog.c | 124 | ||||
| -rw-r--r-- | src/backend/access/transam/xlogfuncs.c | 59 | ||||
| -rw-r--r-- | src/backend/catalog/system_views.sql | 5 | ||||
| -rw-r--r-- | src/backend/postmaster/Makefile | 5 | ||||
| -rw-r--r-- | src/backend/postmaster/bgworker.c | 7 | ||||
| -rw-r--r-- | src/backend/postmaster/checksumhelper.c | 855 | ||||
| -rw-r--r-- | src/backend/postmaster/pgstat.c | 5 | ||||
| -rw-r--r-- | src/backend/replication/basebackup.c | 2 | ||||
| -rw-r--r-- | src/backend/replication/logical/decode.c | 1 | ||||
| -rw-r--r-- | src/backend/storage/ipc/ipci.c | 2 | ||||
| -rw-r--r-- | src/backend/storage/page/README | 3 | ||||
| -rw-r--r-- | src/backend/storage/page/bufpage.c | 6 | ||||
| -rw-r--r-- | src/backend/utils/misc/guc.c | 37 |
14 files changed, 25 insertions, 1102 deletions
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index a31f8b806a8..00741c7b09e 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -17,7 +17,6 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "catalog/pg_control.h" -#include "storage/bufpage.h" #include "utils/guc.h" #include "utils/timestamp.h" @@ -138,18 +137,6 @@ xlog_desc(StringInfo buf, XLogReaderState *record) xlrec.ThisTimeLineID, xlrec.PrevTimeLineID, timestamptz_to_str(xlrec.end_time)); } - else if (info == XLOG_CHECKSUMS) - { - xl_checksum_state xlrec; - - memcpy(&xlrec, rec, sizeof(xl_checksum_state)); - if (xlrec.new_checksumtype == PG_DATA_CHECKSUM_VERSION) - appendStringInfo(buf, "on"); - else if (xlrec.new_checksumtype == PG_DATA_CHECKSUM_INPROGRESS_VERSION) - appendStringInfo(buf, "inprogress"); - else - appendStringInfo(buf, "off"); - } } const char * @@ -195,9 +182,6 @@ xlog_identify(uint8 info) case XLOG_FPI_FOR_HINT: id = "FPI_FOR_HINT"; break; - case XLOG_CHECKSUMS: - id = "CHECKSUMS"; - break; } return id; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 4a47395174c..a51df6b0b90 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -856,7 +856,6 @@ static void SetLatestXTime(TimestampTz xtime); static void SetCurrentChunkStartTime(TimestampTz xtime); static void CheckRequiredParameterValues(void); static void XLogReportParameters(void); -static void XlogChecksums(ChecksumType new_type); static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI, TimeLineID prevTLI); static void LocalSetXLogInsertAllowed(void); @@ -1034,7 +1033,7 @@ XLogInsertRecord(XLogRecData *rdata, Assert(RedoRecPtr < Insert->RedoRecPtr); RedoRecPtr = Insert->RedoRecPtr; } - doPageWrites = (Insert->fullPageWrites || Insert->forcePageWrites || DataChecksumsInProgress()); + doPageWrites = (Insert->fullPageWrites || Insert->forcePageWrites); if (fpw_lsn != InvalidXLogRecPtr && fpw_lsn <= RedoRecPtr && doPageWrites) { @@ -4674,6 +4673,10 @@ ReadControlFile(void) (SizeOfXLogLongPHD - SizeOfXLogShortPHD); CalculateCheckpointSegments(); + + /* Make the initdb settings visible as GUC variables, too */ + SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no", + PGC_INTERNAL, PGC_S_OVERRIDE); } void @@ -4745,90 +4748,12 @@ GetMockAuthenticationNonce(void) * Are checksums enabled for data pages? */ bool -DataChecksumsNeedWrite(void) +DataChecksumsEnabled(void) { Assert(ControlFile != NULL); return (ControlFile->data_checksum_version > 0); } -bool -DataChecksumsNeedVerify(void) -{ - Assert(ControlFile != NULL); - - /* - * Only verify checksums if they are fully enabled in the cluster. In - * inprogress state they are only updated, not verified. - */ - return (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION); -} - -bool -DataChecksumsInProgress(void) -{ - Assert(ControlFile != NULL); - return (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION); -} - -void -SetDataChecksumsInProgress(void) -{ - Assert(ControlFile != NULL); - if (ControlFile->data_checksum_version > 0) - return; - - XlogChecksums(PG_DATA_CHECKSUM_INPROGRESS_VERSION); - - LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - ControlFile->data_checksum_version = PG_DATA_CHECKSUM_INPROGRESS_VERSION; - UpdateControlFile(); - LWLockRelease(ControlFileLock); -} - -void -SetDataChecksumsOn(void) -{ - Assert(ControlFile != NULL); - - LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - - if (ControlFile->data_checksum_version != PG_DATA_CHECKSUM_INPROGRESS_VERSION) - { - LWLockRelease(ControlFileLock); - elog(ERROR, "Checksums not in inprogress mode"); - } - - ControlFile->data_checksum_version = PG_DATA_CHECKSUM_VERSION; - UpdateControlFile(); - LWLockRelease(ControlFileLock); - - XlogChecksums(PG_DATA_CHECKSUM_VERSION); -} - -void -SetDataChecksumsOff(void) -{ - LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - - ControlFile->data_checksum_version = 0; - UpdateControlFile(); - LWLockRelease(ControlFileLock); - - XlogChecksums(0); -} - -/* guc hook */ -const char * -show_data_checksums(void) -{ - if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) - return "on"; - else if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION) - return "inprogress"; - else - return "off"; -} - /* * Returns a fake LSN for unlogged relations. * @@ -7864,16 +7789,6 @@ StartupXLOG(void) CompleteCommitTsInitialization(); /* - * If we reach this point with checksums in inprogress state, we notify - * the user that they need to manually restart the process to enable - * checksums. - */ - if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION) - ereport(WARNING, - (errmsg("checksum state is \"inprogress\" with no worker"), - errhint("Either disable or enable checksums by calling the pg_disable_data_checksums() or pg_enable_data_checksums() functions."))); - - /* * All done with end-of-recovery actions. * * Now allow backends to write WAL and update the control file status in @@ -9627,22 +9542,6 @@ XLogReportParameters(void) } /* - * Log the new state of checksums - */ -static void -XlogChecksums(ChecksumType new_type) -{ - xl_checksum_state xlrec; - - xlrec.new_checksumtype = new_type; - - XLogBeginInsert(); - XLogRegisterData((char *) &xlrec, sizeof(xl_checksum_state)); - - XLogInsert(RM_XLOG_ID, XLOG_CHECKSUMS); -} - -/* * Update full_page_writes in shared memory, and write an * XLOG_FPW_CHANGE record if necessary. * @@ -10070,17 +9969,6 @@ xlog_redo(XLogReaderState *record) /* Keep track of full_page_writes */ lastFullPageWrites = fpw; } - else if (info == XLOG_CHECKSUMS) - { - xl_checksum_state state; - - memcpy(&state, XLogRecGetData(record), sizeof(xl_checksum_state)); - - LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - ControlFile->data_checksum_version = state.new_checksumtype; - UpdateControlFile(); - LWLockRelease(ControlFileLock); - } } #ifdef WAL_DEBUG diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c index b76b2688911..316edbe3c58 100644 --- a/src/backend/access/transam/xlogfuncs.c +++ b/src/backend/access/transam/xlogfuncs.c @@ -24,7 +24,6 @@ #include "catalog/pg_type.h" #include "funcapi.h" #include "miscadmin.h" -#include "postmaster/checksumhelper.h" #include "replication/walreceiver.h" #include "storage/smgr.h" #include "utils/builtins.h" @@ -699,61 +698,3 @@ pg_backup_start_time(PG_FUNCTION_ARGS) PG_RETURN_DATUM(xtime); } - -/* - * Disables checksums for the cluster, unless already disabled. - * - * Has immediate effect - the checksums are set to off right away. - */ -Datum -disable_data_checksums(PG_FUNCTION_ARGS) -{ - /* - * If we don't need to write new checksums, then clearly they are already - * disabled. - */ - if (!DataChecksumsNeedWrite()) - ereport(ERROR, - (errmsg("data checksums already disabled"))); - - ShutdownChecksumHelperIfRunning(); - - SetDataChecksumsOff(); - - PG_RETURN_VOID(); -} - -/* - * Enables checksums for the cluster, unless already enabled. - * - * Supports vacuum-like cost-based throttling, to limit system load. - * Starts a background worker that updates checksums on existing data. - */ -Datum -enable_data_checksums(PG_FUNCTION_ARGS) -{ - int cost_delay = PG_GETARG_INT32(0); - int cost_limit = PG_GETARG_INT32(1); - - if (cost_delay < 0) - ereport(ERROR, - (errmsg("cost delay cannot be less than zero"))); - if (cost_limit <= 0) - ereport(ERROR, - (errmsg("cost limit must be a positive value"))); - - /* - * Allow state change from "off" or from "inprogress", since this is how - * we restart the worker if necessary. - */ - if (DataChecksumsNeedVerify()) - ereport(ERROR, - (errmsg("data checksums already enabled"))); - - SetDataChecksumsInProgress(); - if (!StartChecksumHelperLauncher(cost_delay, cost_limit)) - ereport(ERROR, - (errmsg("failed to start checksum helper process"))); - - PG_RETURN_VOID(); -} diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 85a17a41b98..8cd8bf40ac4 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1027,11 +1027,6 @@ CREATE OR REPLACE FUNCTION pg_stop_backup ( RETURNS SETOF record STRICT VOLATILE LANGUAGE internal as 'pg_stop_backup_v2' PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION pg_enable_data_checksums ( - cost_delay int DEFAULT 0, cost_limit int DEFAULT 100) - RETURNS void STRICT VOLATILE LANGUAGE internal AS 'enable_data_checksums' - PARALLEL RESTRICTED; - -- legacy definition for compatibility with 9.3 CREATE OR REPLACE FUNCTION json_populate_record(base anyelement, from_json json, use_json_as_text boolean DEFAULT false) diff --git a/src/backend/postmaster/Makefile b/src/backend/postmaster/Makefile index ee8f8c1cd33..71c23211b2a 100644 --- a/src/backend/postmaster/Makefile +++ b/src/backend/postmaster/Makefile @@ -12,8 +12,7 @@ subdir = src/backend/postmaster top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = autovacuum.o bgworker.o bgwriter.o checkpointer.o checksumhelper.o \ - fork_process.o pgarch.o pgstat.o postmaster.o startup.o syslogger.o \ - walwriter.o +OBJS = autovacuum.o bgworker.o bgwriter.o checkpointer.o fork_process.o \ + pgarch.o pgstat.o postmaster.o startup.o syslogger.o walwriter.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 19529d77ad6..f651bb49b15 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -20,7 +20,6 @@ #include "pgstat.h" #include "port/atomics.h" #include "postmaster/bgworker_internals.h" -#include "postmaster/checksumhelper.h" #include "postmaster/postmaster.h" #include "replication/logicallauncher.h" #include "replication/logicalworker.h" @@ -130,12 +129,6 @@ static const struct }, { "ApplyWorkerMain", ApplyWorkerMain - }, - { - "ChecksumHelperLauncherMain", ChecksumHelperLauncherMain - }, - { - "ChecksumHelperWorkerMain", ChecksumHelperWorkerMain } }; diff --git a/src/backend/postmaster/checksumhelper.c b/src/backend/postmaster/checksumhelper.c deleted file mode 100644 index 288ab86336f..00000000000 --- a/src/backend/postmaster/checksumhelper.c +++ /dev/null @@ -1,855 +0,0 @@ -/*------------------------------------------------------------------------- - * - * checksumhelper.c - * Background worker to walk the database and write checksums to pages - * - * When enabling data checksums on a database at initdb time, no extra process - * is required as each page is checksummed, and verified, at accesses. When - * enabling checksums on an already running cluster, which was not initialized - * with checksums, this helper worker will ensure that all pages are - * checksummed before verification of the checksums is turned on. - * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * - * IDENTIFICATION - * src/backend/postmaster/checksumhelper.c - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" - -#include "access/heapam.h" -#include "access/htup_details.h" -#include "access/xact.h" -#include "catalog/pg_database.h" -#include "commands/vacuum.h" -#include "common/relpath.h" -#include "miscadmin.h" -#include "pgstat.h" -#include "postmaster/bgworker.h" -#include "postmaster/bgwriter.h" -#include "postmaster/checksumhelper.h" -#include "storage/bufmgr.h" -#include "storage/checksum.h" -#include "storage/lmgr.h" -#include "storage/ipc.h" -#include "storage/procarray.h" -#include "storage/smgr.h" -#include "tcop/tcopprot.h" -#include "utils/lsyscache.h" -#include "utils/ps_status.h" - - -typedef enum -{ - SUCCESSFUL = 0, - ABORTED, - FAILED -} ChecksumHelperResult; - -typedef struct ChecksumHelperShmemStruct -{ - pg_atomic_flag launcher_started; - ChecksumHelperResult success; - bool process_shared_catalogs; - bool abort; - /* Parameter values set on start */ - int cost_delay; - int cost_limit; -} ChecksumHelperShmemStruct; - -/* Shared memory segment for checksumhelper */ -static ChecksumHelperShmemStruct * ChecksumHelperShmem; - -/* Bookkeeping for work to do */ -typedef struct ChecksumHelperDatabase -{ - Oid dboid; - char *dbname; -} ChecksumHelperDatabase; - -typedef struct ChecksumHelperRelation -{ - Oid reloid; - char relkind; -} ChecksumHelperRelation; - -/* Prototypes */ -static List *BuildDatabaseList(void); -static List *BuildRelationList(bool include_shared); -static List *BuildTempTableList(void); -static ChecksumHelperResult ProcessDatabase(ChecksumHelperDatabase * db); -static void launcher_cancel_handler(SIGNAL_ARGS); - -/* - * Main entry point for checksumhelper launcher process. - */ -bool -StartChecksumHelperLauncher(int cost_delay, int cost_limit) -{ - BackgroundWorker bgw; - BackgroundWorkerHandle *bgw_handle; - - if (ChecksumHelperShmem->abort) - { - ereport(ERROR, - (errmsg("could not start checksumhelper: has been cancelled"))); - } - - ChecksumHelperShmem->cost_delay = cost_delay; - ChecksumHelperShmem->cost_limit = cost_limit; - - memset(&bgw, 0, sizeof(bgw)); - bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; - snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ChecksumHelperLauncherMain"); - snprintf(bgw.bgw_name, BGW_MAXLEN, "checksumhelper launcher"); - snprintf(bgw.bgw_type, BGW_MAXLEN, "checksumhelper launcher"); - bgw.bgw_restart_time = BGW_NEVER_RESTART; - bgw.bgw_notify_pid = MyProcPid; - bgw.bgw_main_arg = (Datum) 0; - - if (!pg_atomic_test_set_flag(&ChecksumHelperShmem->launcher_started)) - { - /* Failed to set means somebody else started */ - ereport(ERROR, - (errmsg("could not start checksumhelper: already running"))); - } - - if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) - { - pg_atomic_clear_flag(&ChecksumHelperShmem->launcher_started); - return false; - } - - return true; -} - -/* - * ShutdownChecksumHelperIfRunning - * Request shutdown of the checksumhelper - * - * This does not turn off processing immediately, it signals the checksum - * process to end when done with the current block. - */ -void -ShutdownChecksumHelperIfRunning(void) -{ - /* If the launcher isn't started, there is nothing to shut down */ - if (pg_atomic_unlocked_test_flag(&ChecksumHelperShmem->launcher_started)) - return; - - /* - * We don't need an atomic variable for aborting, setting it multiple - * times will not change the handling. - */ - ChecksumHelperShmem->abort = true; -} - -/* - * ProcessSingleRelationFork - * Enable checksums in a single relation/fork. - * - * Returns true if successful, and false if *aborted*. On error, an actual - * error is raised in the lower levels. - */ -static bool -ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrategy strategy) -{ - BlockNumber numblocks = RelationGetNumberOfBlocksInFork(reln, forkNum); - BlockNumber b; - char activity[NAMEDATALEN * 2 + 128]; - - for (b = 0; b < numblocks; b++) - { - Buffer buf = ReadBufferExtended(reln, forkNum, b, RBM_NORMAL, strategy); - - /* - * Report to pgstat every 100 blocks (so as not to "spam") - */ - if ((b % 100) == 0) - { - snprintf(activity, sizeof(activity) - 1, "processing: %s.%s (%s block %d/%d)", - get_namespace_name(RelationGetNamespace(reln)), RelationGetRelationName(reln), - forkNames[forkNum], b, numblocks); - pgstat_report_activity(STATE_RUNNING, activity); - } - - /* Need to get an exclusive lock before we can flag as dirty */ - LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); - - /* - * Mark the buffer as dirty and force a full page write. We have to - * re-write the page to WAL even if the checksum hasn't changed, - * because if there is a replica it might have a slightly different - * version of the page with an invalid checksum, caused by unlogged - * changes (e.g. hintbits) on the master happening while checksums - * were off. This can happen if there was a valid checksum on the page - * at one point in the past, so only when checksums are first on, then - * off, and then turned on again. - */ - START_CRIT_SECTION(); - MarkBufferDirty(buf); - log_newpage_buffer(buf, false); - END_CRIT_SECTION(); - - UnlockReleaseBuffer(buf); - - /* - * This is the only place where we check if we are asked to abort, the - * abortion will bubble up from here. - */ - if (ChecksumHelperShmem->abort) - return false; - - vacuum_delay_point(); - } - - return true; -} - -/* - * ProcessSingleRelationByOid - * Process a single relation based on oid. - * - * Returns true if successful, and false if *aborted*. On error, an actual error - * is raised in the lower levels. - */ -static bool -ProcessSingleRelationByOid(Oid relationId, BufferAccessStrategy strategy) -{ - Relation rel; - ForkNumber fnum; - bool aborted = false; - - StartTransactionCommand(); - - elog(DEBUG2, "Checksumhelper starting to process relation %d", relationId); - rel = try_relation_open(relationId, AccessShareLock); - if (rel == NULL) - { - /* - * Relation no longer exist. We consider this a success, since there - * are no pages in it that need checksums, and thus return true. - */ - elog(DEBUG1, "Checksumhelper skipping relation %d as it no longer exists", relationId); - CommitTransactionCommand(); - pgstat_report_activity(STATE_IDLE, NULL); - return true; - } - RelationOpenSmgr(rel); - - for (fnum = 0; fnum <= MAX_FORKNUM; fnum++) - { - if (smgrexists(rel->rd_smgr, fnum)) - { - if (!ProcessSingleRelationFork(rel, fnum, strategy)) - { - aborted = true; - break; - } - } - } - relation_close(rel, AccessShareLock); - elog(DEBUG2, "Checksumhelper done with relation %d: %s", - relationId, (aborted ? "aborted" : "finished")); - - CommitTransactionCommand(); - - pgstat_report_activity(STATE_IDLE, NULL); - - return !aborted; -} - -/* - * ProcessDatabase - * Enable checksums in a single database. - * - * We do this by launching a dynamic background worker into this database, and - * waiting for it to finish. We have to do this in a separate worker, since - * each process can only be connected to one database during its lifetime. - */ -static ChecksumHelperResult -ProcessDatabase(ChecksumHelperDatabase * db) -{ - BackgroundWorker bgw; - BackgroundWorkerHandle *bgw_handle; - BgwHandleStatus status; - pid_t pid; - char activity[NAMEDATALEN + 64]; - - ChecksumHelperShmem->success = FAILED; - - memset(&bgw, 0, sizeof(bgw)); - bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; - snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ChecksumHelperWorkerMain"); - snprintf(bgw.bgw_name, BGW_MAXLEN, "checksumhelper worker"); - snprintf(bgw.bgw_type, BGW_MAXLEN, "checksumhelper worker"); - bgw.bgw_restart_time = BGW_NEVER_RESTART; - bgw.bgw_notify_pid = MyProcPid; - bgw.bgw_main_arg = ObjectIdGetDatum(db->dboid); - - if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) - { - ereport(LOG, - (errmsg("failed to start worker for checksumhelper in \"%s\"", - db->dbname))); - return FAILED; - } - - status = WaitForBackgroundWorkerStartup(bgw_handle, &pid); - if (status != BGWH_STARTED) - { - ereport(LOG, - (errmsg("failed to wait for worker startup for checksumhelper in \"%s\"", - db->dbname))); - return FAILED; - } - - ereport(DEBUG1, - (errmsg("started background worker for checksums in \"%s\"", - db->dbname))); - - snprintf(activity, sizeof(activity) - 1, - "Waiting for worker in database %s (pid %d)", db->dbname, pid); - pgstat_report_activity(STATE_RUNNING, activity); - - - status = WaitForBackgroundWorkerShutdown(bgw_handle); - if (status != BGWH_STOPPED) - { - ereport(LOG, - (errmsg("failed to wait for worker shutdown for checksumhelper in \"%s\"", - db->dbname))); - return FAILED; - } - - if (ChecksumHelperShmem->success == ABORTED) - ereport(LOG, - (errmsg("checksumhelper was aborted during processing in \"%s\"", - db->dbname))); - - ereport(DEBUG1, - (errmsg("background worker for checksums in \"%s\" completed", - db->dbname))); - - pgstat_report_activity(STATE_IDLE, NULL); - - return ChecksumHelperShmem->success; -} - -static void -launcher_exit(int code, Datum arg) -{ - ChecksumHelperShmem->abort = false; - pg_atomic_clear_flag(&ChecksumHelperShmem->launcher_started); -} - -static void -launcher_cancel_handler(SIGNAL_ARGS) -{ - ChecksumHelperShmem->abort = true; -} - -static void -WaitForAllTransactionsToFinish(void) -{ - TransactionId waitforxid; - - LWLockAcquire(XidGenLock, LW_SHARED); - waitforxid = ShmemVariableCache->nextXid; - LWLockRelease(XidGenLock); - - while (true) - { - TransactionId oldestxid = GetOldestActiveTransactionId(); - - elog(DEBUG1, "Checking old transactions"); - if (TransactionIdPrecedes(oldestxid, waitforxid)) - { - char activity[64]; - - /* Oldest running xid is older than us, so wait */ - snprintf(activity, sizeof(activity), "Waiting for current transactions to finish (waiting for %d)", waitforxid); - pgstat_report_activity(STATE_RUNNING, activity); - - /* Retry every 5 seconds */ - ResetLatch(MyLatch); - (void) WaitLatch(MyLatch, - WL_LATCH_SET | WL_TIMEOUT, - 5000, - WAIT_EVENT_PG_SLEEP); - } - else - { - pgstat_report_activity(STATE_IDLE, NULL); - return; - } - } -} - -void -ChecksumHelperLauncherMain(Datum arg) -{ - List *DatabaseList; - List *remaining = NIL; - ListCell *lc, - *lc2; - List *CurrentDatabases = NIL; - bool found_failed = false; - - on_shmem_exit(launcher_exit, 0); - - ereport(DEBUG1, - (errmsg("checksumhelper launcher started"))); - - pqsignal(SIGTERM, die); - pqsignal(SIGINT, launcher_cancel_handler); - - BackgroundWorkerUnblockSignals(); - - init_ps_display(pgstat_get_backend_desc(B_CHECKSUMHELPER_LAUNCHER), "", "", ""); - - /* - * Initialize a connection to shared catalogs only. - */ - BackgroundWorkerInitializeConnection(NULL, NULL, 0); - - /* - * Set up so first run processes shared catalogs, but not once in every - * db. - */ - ChecksumHelperShmem->process_shared_catalogs = true; - - /* - * Wait for all existing transactions to finish. This will make sure that - * we can see all tables all databases, so we don't miss any. Anything - * created after this point is known to have checksums on all pages - * already, so we don't have to care about those. - */ - WaitForAllTransactionsToFinish(); - - /* - * Create a database list. We don't need to concern ourselves with - * rebuilding this list during runtime since any database created after - * this process started will be running with checksums turned on from the - * start. - */ - DatabaseList = BuildDatabaseList(); - - /* - * If there are no databases at all to checksum, we can exit immediately - * as there is no work to do. - */ - if (DatabaseList == NIL || list_length(DatabaseList) == 0) - return; - - foreach(lc, DatabaseList) - { - ChecksumHelperDatabase *db = (ChecksumHelperDatabase *) lfirst(lc); - ChecksumHelperResult processing; - - processing = ProcessDatabase(db); - - if (processing == SUCCESSFUL) - { - pfree(db->dbname); - pfree(db); - - if (ChecksumHelperShmem->process_shared_catalogs) - - /* - * Now that one database has completed shared catalogs, we - * don't have to process them again. - */ - ChecksumHelperShmem->process_shared_catalogs = false; - } - else if (processing == FAILED) - { - /* - * Put failed databases on the remaining list. - */ - remaining = lappend(remaining, db); - } - else - /* aborted */ - return; - } - list_free(DatabaseList); - - /* - * remaining now has all databases not yet processed. This can be because - * they failed for some reason, or because the database was dropped - * between us getting the database list and trying to process it. Get a - * fresh list of databases to detect the second case where the database - * was dropped before we had started processing it. If a database still - * exists, but enabling checksums failed then we fail the entire - * checksumming process and exit with an error. - */ - CurrentDatabases = BuildDatabaseList(); - - foreach(lc, remaining) - { - ChecksumHelperDatabase *db = (ChecksumHelperDatabase *) lfirst(lc); - bool found = false; - - foreach(lc2, CurrentDatabases) - { - ChecksumHelperDatabase *db2 = (ChecksumHelperDatabase *) lfirst(lc2); - - if (db->dboid == db2->dboid) - { - found = true; - ereport(WARNING, - (errmsg("failed to enable checksums in \"%s\"", - db->dbname))); - break; - } - } - - if (found) - found_failed = true; - else - { - ereport(LOG, - (errmsg("database \"%s\" has been dropped, skipping", - db->dbname))); - } - - pfree(db->dbname); - pfree(db); - } - list_free(remaining); - - /* Free the extra list of databases */ - foreach(lc, CurrentDatabases) - { - ChecksumHelperDatabase *db = (ChecksumHelperDatabase *) lfirst(lc); - - pfree(db->dbname); - pfree(db); - } - list_free(CurrentDatabases); - - if (found_failed) - { - /* Disable checksums on cluster, because we failed */ - SetDataChecksumsOff(); - ereport(ERROR, - (errmsg("checksumhelper failed to enable checksums in all databases, aborting"))); - } - - /* - * Force a checkpoint to get everything out to disk. - */ - RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_WAIT | CHECKPOINT_IMMEDIATE); - - /* - * Everything has been processed, so flag checksums enabled. - */ - SetDataChecksumsOn(); - - ereport(LOG, - (errmsg("checksums enabled, checksumhelper launcher shutting down"))); -} - -/* - * ChecksumHelperShmemSize - * Compute required space for checksumhelper-related shared memory - */ -Size -ChecksumHelperShmemSize(void) -{ - Size size; - - size = sizeof(ChecksumHelperShmemStruct); - size = MAXALIGN(size); - - return size; -} - -/* - * ChecksumHelperShmemInit - * Allocate and initialize checksumhelper-related shared memory - */ -void -ChecksumHelperShmemInit(void) -{ - bool found; - - ChecksumHelperShmem = (ChecksumHelperShmemStruct *) - ShmemInitStruct("ChecksumHelper Data", - ChecksumHelperShmemSize(), - &found); - - if (!found) - { - MemSet(ChecksumHelperShmem, 0, ChecksumHelperShmemSize()); - pg_atomic_init_flag(&ChecksumHelperShmem->launcher_started); - } -} - -/* - * BuildDatabaseList - * Compile a list of all currently available databases in the cluster - * - * This creates the list of databases for the checksumhelper workers to add - * checksums to. - */ -static List * -BuildDatabaseList(void) -{ - List *DatabaseList = NIL; - Relation rel; - HeapScanDesc scan; - HeapTuple tup; - MemoryContext ctx = CurrentMemoryContext; - MemoryContext oldctx; - - StartTransactionCommand(); - - rel = heap_open(DatabaseRelationId, AccessShareLock); - scan = heap_beginscan_catalog(rel, 0, NULL); - - while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) - { - Form_pg_database pgdb = (Form_pg_database) GETSTRUCT(tup); - ChecksumHelperDatabase *db; - - oldctx = MemoryContextSwitchTo(ctx); - - db = (ChecksumHelperDatabase *) palloc(sizeof(ChecksumHelperDatabase)); - - db->dboid = HeapTupleGetOid(tup); - db->dbname = pstrdup(NameStr(pgdb->datname)); - - DatabaseList = lappend(DatabaseList, db); - - MemoryContextSwitchTo(oldctx); - } - - heap_endscan(scan); - heap_close(rel, AccessShareLock); - - CommitTransactionCommand(); - - return DatabaseList; -} - -/* - * BuildRelationList - * Compile a list of all relations in the database - * - * If shared is true, both shared relations and local ones are returned, else - * all non-shared relations are returned. - * Temp tables are not included. - */ -static List * -BuildRelationList(bool include_shared) -{ - List *RelationList = NIL; - Relation rel; - HeapScanDesc scan; - HeapTuple tup; - MemoryContext ctx = CurrentMemoryContext; - MemoryContext oldctx; - - StartTransactionCommand(); - - rel = heap_open(RelationRelationId, AccessShareLock); - scan = heap_beginscan_catalog(rel, 0, NULL); - - while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) - { - Form_pg_class pgc = (Form_pg_class) GETSTRUCT(tup); - ChecksumHelperRelation *relentry; - - if (pgc->relpersistence == 't') - continue; - - if (pgc->relisshared && !include_shared) - continue; - - /* - * Foreign tables have by definition no local storage that can be - * checksummed, so skip. - */ - if (pgc->relkind == RELKIND_FOREIGN_TABLE) - continue; - - oldctx = MemoryContextSwitchTo(ctx); - relentry = (ChecksumHelperRelation *) palloc(sizeof(ChecksumHelperRelation)); - - relentry->reloid = HeapTupleGetOid(tup); - relentry->relkind = pgc->relkind; - - RelationList = lappend(RelationList, relentry); - - MemoryContextSwitchTo(oldctx); - } - - heap_endscan(scan); - heap_close(rel, AccessShareLock); - - CommitTransactionCommand(); - - return RelationList; -} - -/* - * BuildTempTableList - * Compile a list of all temporary tables in database - * - * Returns a List of oids. - */ -static List * -BuildTempTableList(void) -{ - List *RelationList = NIL; - Relation rel; - HeapScanDesc scan; - HeapTuple tup; - MemoryContext ctx = CurrentMemoryContext; - MemoryContext oldctx; - - StartTransactionCommand(); - - rel = heap_open(RelationRelationId, AccessShareLock); - scan = heap_beginscan_catalog(rel, 0, NULL); - - while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) - { - Form_pg_class pgc = (Form_pg_class) GETSTRUCT(tup); - - if (pgc->relpersistence != 't') - continue; - - oldctx = MemoryContextSwitchTo(ctx); - RelationList = lappend_oid(RelationList, HeapTupleGetOid(tup)); - MemoryContextSwitchTo(oldctx); - } - - heap_endscan(scan); - heap_close(rel, AccessShareLock); - - CommitTransactionCommand(); - - return RelationList; -} - -/* - * Main function for enabling checksums in a single database - */ -void -ChecksumHelperWorkerMain(Datum arg) -{ - Oid dboid = DatumGetObjectId(arg); - List *RelationList = NIL; - List *InitialTempTableList = NIL; - ListCell *lc; - BufferAccessStrategy strategy; - bool aborted = false; - - pqsignal(SIGTERM, die); - - BackgroundWorkerUnblockSignals(); - - init_ps_display(pgstat_get_backend_desc(B_CHECKSUMHELPER_WORKER), "", "", ""); - - ereport(DEBUG1, - (errmsg("checksum worker starting for database oid %d", dboid))); - - BackgroundWorkerInitializeConnectionByOid(dboid, InvalidOid, BGWORKER_BYPASS_ALLOWCONN); - - /* - * Get a list of all temp tables present as we start in this database. We - * need to wait until they are all gone until we are done, since we cannot - * access those files and modify them. - */ - InitialTempTableList = BuildTempTableList(); - - /* - * Enable vacuum cost delay, if any. - */ - VacuumCostDelay = ChecksumHelperShmem->cost_delay; - VacuumCostLimit = ChecksumHelperShmem->cost_limit; - VacuumCostActive = (VacuumCostDelay > 0); - VacuumCostBalance = 0; - VacuumPageHit = 0; - VacuumPageMiss = 0; - VacuumPageDirty = 0; - - /* - * Create and set the vacuum strategy as our buffer strategy. - */ - strategy = GetAccessStrategy(BAS_VACUUM); - - RelationList = BuildRelationList(ChecksumHelperShmem->process_shared_catalogs); - foreach(lc, RelationList) - { - ChecksumHelperRelation *rel = (ChecksumHelperRelation *) lfirst(lc); - - if (!ProcessSingleRelationByOid(rel->reloid, strategy)) - { - aborted = true; - break; - } - } - list_free_deep(RelationList); - - if (aborted) - { - ChecksumHelperShmem->success = ABORTED; - ereport(DEBUG1, - (errmsg("checksum worker aborted in database oid %d", dboid))); - return; - } - - /* - * Wait for all temp tables that existed when we started to go away. This - * is necessary since we cannot "reach" them to enable checksums. Any temp - * tables created after we started will already have checksums in them - * (due to the inprogress state), so those are safe. - */ - while (true) - { - List *CurrentTempTables; - ListCell *lc; - int numleft; - char activity[64]; - - CurrentTempTables = BuildTempTableList(); - numleft = 0; - foreach(lc, InitialTempTableList) - { - if (list_member_oid(CurrentTempTables, lfirst_oid(lc))) - numleft++; - } - list_free(CurrentTempTables); - - if (numleft == 0) - break; - - /* At least one temp table left to wait for */ - snprintf(activity, sizeof(activity), "Waiting for %d temp tables to be removed", numleft); - pgstat_report_activity(STATE_RUNNING, activity); - - /* Retry every 5 seconds */ - ResetLatch(MyLatch); - (void) WaitLatch(MyLatch, - WL_LATCH_SET | WL_TIMEOUT, - 5000, - WAIT_EVENT_PG_SLEEP); - } - - list_free(InitialTempTableList); - - ChecksumHelperShmem->success = SUCCESSFUL; - ereport(DEBUG1, - (errmsg("checksum worker completed in database oid %d", dboid))); -} diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 83328a27662..96ba2163878 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -4125,11 +4125,6 @@ pgstat_get_backend_desc(BackendType backendType) case B_WAL_WRITER: backendDesc = "walwriter"; break; - case B_CHECKSUMHELPER_LAUNCHER: - backendDesc = "checksumhelper launcher"; - break; - case B_CHECKSUMHELPER_WORKER: - backendDesc = "checksumhelper worker"; } return backendDesc; diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index babf85a6eaf..bfbd1fd3821 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -1384,7 +1384,7 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf _tarWriteHeader(tarfilename, NULL, statbuf, false); - if (!noverify_checksums && DataChecksumsNeedVerify()) + if (!noverify_checksums && DataChecksumsEnabled()) { char *filename; diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index 30d80e7c542..59c003de9ce 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -199,7 +199,6 @@ DecodeXLogOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) case XLOG_FPW_CHANGE: case XLOG_FPI_FOR_HINT: case XLOG_FPI: - case XLOG_CHECKSUMS: break; default: elog(ERROR, "unexpected RM_XLOG_ID record type: %u", info); diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 853e1e472f6..0c86a581c03 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -27,7 +27,6 @@ #include "postmaster/autovacuum.h" #include "postmaster/bgworker_internals.h" #include "postmaster/bgwriter.h" -#include "postmaster/checksumhelper.h" #include "postmaster/postmaster.h" #include "replication/logicallauncher.h" #include "replication/slot.h" @@ -262,7 +261,6 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) WalSndShmemInit(); WalRcvShmemInit(); ApplyLauncherShmemInit(); - ChecksumHelperShmemInit(); /* * Set up other modules that need some shared memory space diff --git a/src/backend/storage/page/README b/src/backend/storage/page/README index f873fb0eea1..5127d98da37 100644 --- a/src/backend/storage/page/README +++ b/src/backend/storage/page/README @@ -9,8 +9,7 @@ have a very low measured incidence according to research on large server farms, http://www.cs.toronto.edu/~bianca/papers/sigmetrics09.pdf, discussed 2010/12/22 on -hackers list. -Checksums can be enabled at initdb time, but can also be turned on and off -using pg_enable_data_checksums()/pg_disable_data_checksums() at runtime. +Current implementation requires this be enabled system-wide at initdb time. The checksum is not valid at all times on a data page!! The checksum is valid when the page leaves the shared pool and is checked diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 790e4b860ad..dfbda5458fd 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -93,7 +93,7 @@ PageIsVerified(Page page, BlockNumber blkno) */ if (!PageIsNew(page)) { - if (DataChecksumsNeedVerify()) + if (DataChecksumsEnabled()) { checksum = pg_checksum_page((char *) page, blkno); @@ -1168,7 +1168,7 @@ PageSetChecksumCopy(Page page, BlockNumber blkno) static char *pageCopy = NULL; /* If we don't need a checksum, just return the passed-in data */ - if (PageIsNew(page) || !DataChecksumsNeedWrite()) + if (PageIsNew(page) || !DataChecksumsEnabled()) return (char *) page; /* @@ -1195,7 +1195,7 @@ void PageSetChecksumInplace(Page page, BlockNumber blkno) { /* If we don't need a checksum, just return */ - if (PageIsNew(page) || !DataChecksumsNeedWrite()) + if (PageIsNew(page) || !DataChecksumsEnabled()) return; ((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 8441837e0ff..fa92ce2e683 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -32,7 +32,6 @@ #include "access/transam.h" #include "access/twophase.h" #include "access/xact.h" -#include "access/xlog.h" #include "access/xlog_internal.h" #include "catalog/namespace.h" #include "catalog/pg_authid.h" @@ -69,7 +68,6 @@ #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/bufmgr.h" -#include "storage/checksum.h" #include "storage/dsm_impl.h" #include "storage/standby.h" #include "storage/fd.h" @@ -423,17 +421,6 @@ static const struct config_enum_entry password_encryption_options[] = { }; /* - * data_checksum used to be a boolean, but was only set by initdb so there is - * no need to support variants of boolean input. - */ -static const struct config_enum_entry data_checksum_options[] = { - {"on", DATA_CHECKSUMS_ON, true}, - {"off", DATA_CHECKSUMS_OFF, true}, - {"inprogress", DATA_CHECKSUMS_INPROGRESS, true}, - {NULL, 0, false} -}; - -/* * Options for enum values stored in other modules */ extern const struct config_enum_entry wal_level_options[]; @@ -528,7 +515,7 @@ static int max_identifier_length; static int block_size; static int segment_size; static int wal_block_size; -static int data_checksums_tmp; /* only accessed locally! */ +static bool data_checksums; static bool integer_datetimes; static bool assert_enabled; @@ -1698,6 +1685,17 @@ static struct config_bool ConfigureNamesBool[] = }, { + {"data_checksums", PGC_INTERNAL, PRESET_OPTIONS, + gettext_noop("Shows whether data checksums are turned on for this cluster."), + NULL, + GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE + }, + &data_checksums, + false, + NULL, NULL, NULL + }, + + { {"syslog_sequence_numbers", PGC_SIGHUP, LOGGING_WHERE, gettext_noop("Add sequence number to syslog messages to avoid duplicate suppression."), NULL @@ -4129,17 +4127,6 @@ static struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, - { - {"data_checksums", PGC_INTERNAL, PRESET_OPTIONS, - gettext_noop("Shows whether data checksums are turned on for this cluster."), - NULL, - GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE - }, - &data_checksums_tmp, - DATA_CHECKSUMS_OFF, data_checksum_options, - NULL, NULL, show_data_checksums - }, - /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL |