Create a GUC parameter max_files_per_process that is a configurable

upper limit on what we will believe from sysconf(_SC_OPEN_MAX).  The
default value is 1000, so that under ordinary conditions it won't
affect the behavior.  But on platforms where the kernel promises far
more than it can deliver, this can be used to prevent running out of
file descriptors.  See numerous past discussions, eg, pgsql-hackers
around 23-Dec-2000.

Author: tglsfdc

doc/src/sgml/runtime.sgml

@@ -1,5 +1,5 @@
 <!--
-$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.85 2001/09/23 21:52:36 petere Exp $
+$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.86 2001/09/30 18:57:45 tgl Exp $
 -->
 
 <Chapter Id="runtime">
@@ -1218,6 +1218,26 @@ dynamic_library_path = '/usr/local/lib/postgresql:/home/my_project/lib:$libdir'
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><varname>MAX_FILES_PER_PROCESS</varname> (<type>integer</type>)</term>
+      <listitem>
+       <para>
+        Sets the maximum number of simultaneously open files in each server
+	process.  The default is 1000.  The limit actually used by the code
+	is the smaller of this setting and the result of
+	<literal>sysconf(_SC_OPEN_MAX)</literal>.
+	Therefore, on systems where sysconf returns a reasonable limit,
+	you don't need to worry about this setting.  But on some platforms
+	(notably, most BSD systems), sysconf returns a value that is much
+	larger than the system can really support when a large number of
+	processes all try to open that many files.  If you find yourself
+	seeing <quote>Too many open files</> failures, try reducing this
+	setting.
+        This option can only be set at server start.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><varname>MAX_FSM_RELATIONS</varname> (<type>integer</type>)</term>
       <listitem>

src/backend/storage/file/fd.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.83 2001/08/04 19:42:34 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.84 2001/09/30 18:57:45 tgl Exp $
  *
  * NOTES:
  *
@@ -69,7 +69,7 @@
  *
  * (Even though most dynamic loaders now use dlopen(3) or the
  * equivalent, the OS must still open several files to perform the
- * dynamic loading.  Keep this here.)
+ * dynamic loading.  And stdin/stdout/stderr count too.  Keep this here.)
  */
 #ifndef RESERVE_FOR_LD
 #define RESERVE_FOR_LD	10
@@ -87,6 +87,14 @@
 #define FD_MINFREE 10
 #endif
 
+/*
+ * A number of platforms return values for sysconf(_SC_OPEN_MAX) that are
+ * far beyond what they can really support.  This GUC parameter limits what
+ * we will believe.
+ */
+int max_files_per_process = 1000;
+
+
 /* Debugging.... */
 
 #ifdef FDDEBUG
@@ -281,29 +289,46 @@ pg_nofile(void)
 {
 	static long no_files = 0;
 
+	/* need do this calculation only once */
 	if (no_files == 0)
 	{
-		/* need do this calculation only once */
-#ifndef HAVE_SYSCONF
-		no_files = (long) NOFILE;
-#else
+		/*
+		 * Ask the system what its files-per-process limit is.
+		 */
+#ifdef HAVE_SYSCONF
 		no_files = sysconf(_SC_OPEN_MAX);
-		if (no_files == -1)
+		if (no_files <= 0)
 		{
-/* tweak for Hurd, which does not support NOFILE */
 #ifdef NOFILE
-			elog(DEBUG, "pg_nofile: Unable to get _SC_OPEN_MAX using sysconf(); using %d", NOFILE);
 			no_files = (long) NOFILE;
 #else
-			elog(FATAL, "pg_nofile: Unable to get _SC_OPEN_MAX using sysconf() and NOFILE is undefined");
+			no_files = (long) max_files_per_process;
 #endif
+			elog(DEBUG, "pg_nofile: sysconf(_SC_OPEN_MAX) failed; using %ld",
+				 no_files);
 		}
+#else /* !HAVE_SYSCONF */
+#ifdef NOFILE
+		no_files = (long) NOFILE;
+#else
+		no_files = (long) max_files_per_process;
 #endif
+#endif /* HAVE_SYSCONF */
+
+		/*
+		 * Some platforms return hopelessly optimistic values.  Apply a
+		 * configurable upper limit.
+		 */
+		if (no_files > (long) max_files_per_process)
+			no_files = (long) max_files_per_process;
 
+		/*
+		 * Make sure we have enough to get by after reserving some for LD.
+		 */
 		if ((no_files - RESERVE_FOR_LD) < FD_MINFREE)
-			elog(FATAL, "pg_nofile: insufficient File Descriptors in postmaster to start backend (%ld).\n"
-				 "                   O/S allows %ld, Postmaster reserves %d, We need %d (MIN) after that.",
-				 no_files - RESERVE_FOR_LD, no_files, RESERVE_FOR_LD, FD_MINFREE);
+			elog(FATAL, "pg_nofile: insufficient file descriptors available to start backend.\n"
+				 "\tSystem allows %ld, we need at least %d.",
+				 no_files, RESERVE_FOR_LD + FD_MINFREE);
 
 		no_files -= RESERVE_FOR_LD;
 	}

src/backend/utils/misc/guc.c

@@ -4,7 +4,7 @@
  * Support for grand unified configuration scheme, including SET
  * command, configuration file, and command line options.
  *
- * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.53 2001/09/29 04:02:25 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.54 2001/09/30 18:57:45 tgl Exp $
  *
  * Copyright 2000 by PostgreSQL Global Development Group
  * Written by Peter Eisentraut <peter_e@gmx.net>.
@@ -31,6 +31,7 @@
 #include "optimizer/paths.h"
 #include "optimizer/planmain.h"
 #include "parser/parse_expr.h"
+#include "storage/fd.h"
 #include "storage/freespace.h"
 #include "storage/lock.h"
 #include "storage/proc.h"
@@ -302,6 +303,9 @@ static struct config_int
 	{"vacuum_mem", PGC_USERSET, &VacuumMem,
 	8192, 1024, INT_MAX, NULL, NULL},
 
+	{"max_files_per_process", PGC_BACKEND, &max_files_per_process,
+	1000, 25, INT_MAX, NULL, NULL},
+
 	{"debug_level", PGC_USERSET, &DebugLvl,
 	0, 0, 16, NULL, NULL},
 

src/backend/utils/misc/postgresql.conf.sample

@@ -176,10 +176,11 @@
 #
 #dynamic_library_path = '$libdir'
 #australian_timezones = false
-#authentication_timeout = 60  # min 1, max 600
+#authentication_timeout = 60    # min 1, max 600
 #deadlock_timeout = 1000
 #default_transaction_isolation = 'read committed'
-#max_expr_depth = 10000 # min 10
+#max_expr_depth = 10000         # min 10
+#max_files_per_process = 1000   # min 25
 #password_encryption = false
 #sql_inheritance = true
 #transform_null_equals = false

src/include/storage/fd.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: fd.h,v 1.30 2001/06/11 04:12:29 tgl Exp $
+ * $Id: fd.h,v 1.31 2001/09/30 18:57:45 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -43,6 +43,11 @@ typedef char *FileName;
 
 typedef int File;
 
+
+/* GUC parameter */
+extern int max_files_per_process;
+
+
 /*
  * prototypes for functions in fd.c
  */