Rework shutdown callback of archiver modules

As currently designed, with a callback registered in a ERROR_CLEANUP
block, the shutdown callback would get called twice when updating
archive_library on SIGHUP, which is something that we want to avoid to
ease the life of extension writers.

Anyway, an ERROR in the archiver process is treated as a FATAL, stopping
it immediately, hence there is no need for a ERROR_CLEANUP block.
Instead of that, the shutdown callback is not called upon
before_shmem_exit(), giving to the modules the opportunity to do any
cleanup actions before the server shuts down its subsystems.

While on it, this commit adds some testing coverage for the shutdown
callback.  Neither shell_archive nor basic_archive have been using it,
and one is added to shell_archive, whose trigger is checked in a TAP
test through a shutdown sequence.

Author: Nathan Bossart, Bharath Rupireddy
Reviewed-by: Kyotaro Horiguchi, Michael Paquier
Discussion: https://postgr.es/m/20221015221328.GB1821022@nathanxps13
Backpatch-through: 15

Author: michaelpq

doc/src/sgml/config.sgml

@@ -3625,8 +3625,9 @@ include_dir 'conf.d'
         The library to use for archiving completed WAL file segments.  If set to
         an empty string (the default), archiving via shell is enabled, and
         <xref linkend="guc-archive-command"/> is used.  Otherwise, the specified
-        shared library is used for archiving.  For more information, see
-        <xref linkend="backup-archiving-wal"/> and
+        shared library is used for archiving. The WAL archiver process is
+        restarted by the postmaster when this parameter changes. For more
+        information, see <xref linkend="backup-archiving-wal"/> and
         <xref linkend="archive-modules"/>.
        </para>
        <para>

src/backend/postmaster/pgarch.c

@@ -144,7 +144,7 @@ static void pgarch_die(int code, Datum arg);
 static void HandlePgArchInterrupts(void);
 static int	ready_file_comparator(Datum a, Datum b, void *arg);
 static void LoadArchiveLibrary(void);
-static void call_archive_module_shutdown_callback(int code, Datum arg);
+static void pgarch_call_module_shutdown_cb(int code, Datum arg);
 
 /* Report shared memory space needed by PgArchShmemInit */
 Size
@@ -252,13 +252,7 @@ PgArchiverMain(void)
 	/* Load the archive_library. */
 	LoadArchiveLibrary();
 
-	PG_ENSURE_ERROR_CLEANUP(call_archive_module_shutdown_callback, 0);
-	{
-		pgarch_MainLoop();
-	}
-	PG_END_ENSURE_ERROR_CLEANUP(call_archive_module_shutdown_callback, 0);
-
-	call_archive_module_shutdown_callback(0, 0);
+	pgarch_MainLoop();
 
 	proc_exit(0);
 }
@@ -803,19 +797,14 @@ HandlePgArchInterrupts(void)
 
 		if (archiveLibChanged)
 		{
-			/*
-			 * Call the currently loaded archive module's shutdown callback,
-			 * if one is defined.
-			 */
-			call_archive_module_shutdown_callback(0, 0);
-
 			/*
 			 * Ideally, we would simply unload the previous archive module and
 			 * load the new one, but there is presently no mechanism for
 			 * unloading a library (see the comment above
 			 * internal_load_library()).  To deal with this, we simply restart
 			 * the archiver.  The new archive module will be loaded when the
-			 * new archiver process starts up.
+			 * new archiver process starts up.  Note that this triggers the
+			 * module's shutdown callback, if defined.
 			 */
 			ereport(LOG,
 					(errmsg("restarting archiver process because value of "
@@ -858,15 +847,15 @@ LoadArchiveLibrary(void)
 	if (ArchiveContext.archive_file_cb == NULL)
 		ereport(ERROR,
 				(errmsg("archive modules must register an archive callback")));
+
+	before_shmem_exit(pgarch_call_module_shutdown_cb, 0);
 }
 
 /*
- * call_archive_module_shutdown_callback
- *
- * Calls the loaded archive module's shutdown callback, if one is defined.
+ * Call the shutdown callback of the loaded archive module, if defined.
  */
 static void
-call_archive_module_shutdown_callback(int code, Datum arg)
+pgarch_call_module_shutdown_cb(int code, Datum arg)
 {
 	if (ArchiveContext.shutdown_cb != NULL)
 		ArchiveContext.shutdown_cb();

src/backend/postmaster/shell_archive.c

@@ -23,6 +23,7 @@
 
 static bool shell_archive_configured(void);
 static bool shell_archive_file(const char *file, const char *path);
+static void shell_archive_shutdown(void);
 
 void
 shell_archive_init(ArchiveModuleCallbacks *cb)
@@ -31,6 +32,7 @@ shell_archive_init(ArchiveModuleCallbacks *cb)
 
 	cb->check_configured_cb = shell_archive_configured;
 	cb->archive_file_cb = shell_archive_file;
+	cb->shutdown_cb = shell_archive_shutdown;
 }
 
 static bool
@@ -156,3 +158,9 @@ shell_archive_file(const char *file, const char *path)
 	elog(DEBUG1, "archived write-ahead log file \"%s\"", file);
 	return true;
 }
+
+static void
+shell_archive_shutdown(void)
+{
+	elog(DEBUG1, "archiver process shutting down");
+}

src/test/recovery/t/020_archive_status.pl

@@ -234,4 +234,18 @@
 	".done files created after archive success with archive_mode=always on standby"
 );
 
+# Check that the archiver process calls the shell archive module's shutdown
+# callback.
+$standby2->append_conf('postgresql.conf', "log_min_messages = debug1");
+$standby2->reload;
+
+# Run a query to make sure that the reload has taken effect.
+$standby2->safe_psql('postgres', q{SELECT 1});
+my $log_location = -s $standby2->logfile;
+
+$standby2->stop;
+my $logfile = slurp_file($standby2->logfile, $log_location);
+ok( $logfile =~ qr/archiver process shutting down/,
+	'check shutdown callback of shell archive module');
+
 done_testing();