Improved parallel make support

Replace for loops in makefiles with proper dependencies.  Parallel
make can now span across directories.  Also, make -k and make -q work
properly.

GNU make 3.80 or newer is now required.

Author: petere

GNUmakefile.in

@@ -8,57 +8,39 @@ subdir =
 top_builddir = .
 include $(top_builddir)/src/Makefile.global
 
+$(call recurse,all install,src config)
+
 all:
-	$(MAKE) -C src all
-	$(MAKE) -C config all
-	@echo "All of PostgreSQL successfully made. Ready to install."
+	+@echo "All of PostgreSQL successfully made. Ready to install."
 
 docs:
 	$(MAKE) -C doc all
 
+$(call recurse,world,doc src config contrib,all)
 world:
-	$(MAKE) -C doc all
-	$(MAKE) -C src all
-	$(MAKE) -C config all
-	$(MAKE) -C contrib all
-	@echo "PostgreSQL, contrib, and documentation successfully made. Ready to install."
+	+@echo "PostgreSQL, contrib, and documentation successfully made. Ready to install."
 
 html man:
 	$(MAKE) -C doc $@
 
 install:
-	$(MAKE) -C src $@
-	$(MAKE) -C config $@
-	@echo "PostgreSQL installation complete."
+	+@echo "PostgreSQL installation complete."
 
 install-docs:
 	$(MAKE) -C doc install
 
+$(call recurse,install-world,doc src config contrib,install)
 install-world:
-	$(MAKE) -C doc install
-	$(MAKE) -C src install
-	$(MAKE) -C config install
-	$(MAKE) -C contrib install
-	@echo "PostgreSQL, contrib, and documentation installation complete."
+	+@echo "PostgreSQL, contrib, and documentation installation complete."
 
-installdirs uninstall coverage:
-	$(MAKE) -C doc $@
-	$(MAKE) -C src $@
-	$(MAKE) -C config $@
+$(call recurse,installdirs uninstall coverage,doc src config)
 
-distprep:
-	$(MAKE) -C doc $@
-	$(MAKE) -C src $@
-	$(MAKE) -C config $@
-	$(MAKE) -C contrib $@
+$(call recurse,distprep,doc src config contrib)
 
 # clean, distclean, etc should apply to contrib too, even though
 # it's not built by default
+$(call recurse,clean,doc contrib src config)
 clean:
-	$(MAKE) -C doc $@
-	$(MAKE) -C contrib $@
-	$(MAKE) -C src $@
-	$(MAKE) -C config $@
 # Garbage from autoconf:
 	@rm -rf autom4te.cache/
 
@@ -78,11 +60,7 @@ check: all
 check installcheck installcheck-parallel:
 	$(MAKE) -C src/test $@
 
-installcheck-world:
-	$(MAKE) -C src/test installcheck
-	$(MAKE) -C src/pl installcheck
-	$(MAKE) -C src/interfaces/ecpg installcheck
-	$(MAKE) -C contrib installcheck
+$(call recurse,installcheck-world,src/test src/pl src/interfaces/ecpg contrib,installcheck)
 
 GNUmakefile: GNUmakefile.in $(top_builddir)/config.status
 	./config.status $@
@@ -143,12 +121,4 @@ distcheck: dist
 	rm -rf $(distdir) $(dummy)
 	@echo "Distribution integrity checks out."
 
-.PHONY: dist distdir distcheck docs install-docs
-
-
-# Temporary measure to explore whether we can start requiring GNU make
-# 3.80.  That version also happens to be the version where the
-# .VARIABLES variable was introduced, so this is a simple check.
-ifndef .VARIABLES
-$(warning warning: GNU make 3.80 or newer might become required soon. You are using version $(MAKE_VERSION).)
-endif
+.PHONY: dist distdir distcheck docs install-docs world install-world installcheck-world

contrib/Makefile

@@ -63,14 +63,4 @@ endif
 #		start-scripts	\ (does not have a makefile)
 
 
-all install installdirs uninstall distprep clean distclean maintainer-clean:
-	@for dir in $(SUBDIRS); do \
-		$(MAKE) -C $$dir $@ || exit; \
-	done
-
-# We'd like check operations to run all the subtests before failing.
-check installcheck:
-	@CHECKERR=0; for dir in $(SUBDIRS); do \
-		$(MAKE) -C $$dir $@ || CHECKERR=$$?; \
-	done; \
-	exit $$CHECKERR
+$(recurse)

contrib/dblink/Makefile

@@ -4,6 +4,7 @@ MODULE_big = dblink
 PG_CPPFLAGS = -I$(libpq_srcdir)
 OBJS	= dblink.o
 SHLIB_LINK = $(libpq)
+SHLIB_PREREQS = submake-libpq
 
 DATA_built = dblink.sql 
 DATA = uninstall_dblink.sql 

doc/src/sgml/installation.sgml

@@ -68,8 +68,8 @@ su - postgres
        <primary>make</primary>
       </indexterm>
 
-      <acronym>GNU</> <application>make</> is required; other
-      <application>make</> programs will <emphasis>not</> work.
+      <acronym>GNU</> <application>make</> version 3.80 or newer is required; other
+      <application>make</> programs or older <acronym>GNU</> <application>make</> versions will <emphasis>not</> work.
       <acronym>GNU</> <application>make</> is often installed under
       the name <filename>gmake</filename>; this document will always
       refer to it by that name. (On some systems
@@ -79,7 +79,6 @@ su - postgres
 <screen>
 <userinput>gmake --version</userinput>
 </screen>
-      It is recommended to use version 3.79.1 or later.
      </para>
     </listitem>
 

src/Makefile

@@ -12,20 +12,21 @@ subdir = src
 top_builddir = ..
 include Makefile.global
 
+SUBDIRS = \
+	port \
+	timezone \
+	backend \
+	backend/utils/mb/conversion_procs \
+	backend/snowball \
+	include \
+	interfaces \
+	backend/replication/libpqwalreceiver \
+	bin \
+	pl \
+	makefiles \
+	test/regress
 
-all install installdirs uninstall distprep:
-	$(MAKE) -C port $@
-	$(MAKE) -C timezone $@
-	$(MAKE) -C backend $@
-	$(MAKE) -C backend/utils/mb/conversion_procs $@
-	$(MAKE) -C backend/snowball $@
-	$(MAKE) -C include $@
-	$(MAKE) -C interfaces $@
-	$(MAKE) -C backend/replication/libpqwalreceiver $@
-	$(MAKE) -C bin $@
-	$(MAKE) -C pl $@
-	$(MAKE) -C makefiles $@
-	$(MAKE) -C test/regress $@
+$(recurse)
 
 install: install-local
 
@@ -46,31 +47,11 @@ uninstall-local:
 	rm -f $(addprefix '$(DESTDIR)$(pgxsdir)/$(subdir)'/, Makefile.global Makefile.port Makefile.shlib nls-global.mk)
 
 clean:
-	$(MAKE) -C port $@
-	$(MAKE) -C timezone $@
-	$(MAKE) -C backend $@
-	$(MAKE) -C backend/snowball $@
-	$(MAKE) -C include $@
-	$(MAKE) -C interfaces $@
-	$(MAKE) -C backend/replication/libpqwalreceiver $@
-	$(MAKE) -C bin $@
-	$(MAKE) -C pl $@
-	$(MAKE) -C makefiles $@
 	$(MAKE) -C test $@
 	$(MAKE) -C tutorial NO_PGXS=1 $@
 	$(MAKE) -C test/thread $@
 
 distclean maintainer-clean:
-	$(MAKE) -C port $@
-	$(MAKE) -C timezone $@
-	$(MAKE) -C backend $@
-	$(MAKE) -C backend/snowball $@
-	$(MAKE) -C include $@
-	$(MAKE) -C interfaces $@
-	$(MAKE) -C backend/replication/libpqwalreceiver $@
-	$(MAKE) -C bin $@
-	$(MAKE) -C pl $@
-	$(MAKE) -C makefiles $@
 	$(MAKE) -C test $@
 	$(MAKE) -C tutorial NO_PGXS=1 $@
 	$(MAKE) -C test/thread $@

src/Makefile.global.in

@@ -18,7 +18,9 @@
 #
 # Meta configuration
 
-.PHONY: all install install-strip installdirs uninstall clean distclean maintainer-clean distprep check installcheck maintainer-check coverage html man installcheck-parallel world install-world installcheck-world
+standard_targets = all install installdirs uninstall distprep clean distclean maintainer-clean coverage check installcheck
+
+.PHONY: $(standard_targets) install-strip maintainer-check html man installcheck-parallel
 
 # make `all' the default target
 all:
@@ -538,6 +540,48 @@ install-strip:
 	    install
 
 
+##########################################################################
+#
+# Recursive make support
+# ----------------------
+# Instead of recursing through subdirectories with a for loop or
+# repeated $(MAKE) -C whatever calls, this is a little smarter: it
+# allows parallel make across directories and lets make -k and -q work
+# correctly.
+
+# This function is only for internal use below.  It should be called
+# with $(eval).  It will set up a target so that it recurses into
+# subdirectories.
+# $1: target name, e.g., all
+# $2: list of subdirs
+# $3: target to run in subdir, usually same as $1
+define _create_recursive_target
+.PHONY: $(patsubst %,$(1)-%-recursive,$(2))
+$(1): $(patsubst %,$(1)-%-recursive,$(2))
+$(2:%=$(1)-%-recursive):
+	$$(MAKE) -C $$(patsubst $(1)-%-recursive,%,$$@) $(3)
+endef
+# Note that the use of $$ on the last line above is important; we want
+# those variables/functions to be evaluated when the rule is run, not
+# when the $(eval) is run to create the rule.  In the case of
+# $$(MAKE), this is necessary to get make -q working.
+
+# Call this function in a makefile.  In the normal case it doesn't
+# need any arguments.
+# $1: targets to make recursive (defaults to list of standard targets)
+# $2: list of subdirs (defaults to SUBDIRS variable)
+# $3: target to run in subdir (defaults to $1)
+recurse = $(foreach target,$(if $1,$1,$(standard_targets)),$(eval $(call _create_recursive_target,$(target),$(if $2,$2,$(SUBDIRS)),$(if $3,$3,$(target)))))
+
+# We need the $(eval) function and order-only prerequisites, which are
+# available in GNU make 3.80.  That also happens to be the version
+# where the .VARIABLES variable was introduced, so this is a simple
+# check.
+ifndef .VARIABLES
+$(error GNU make 3.80 or newer is required.  You are using version $(MAKE_VERSION))
+endif
+
+
 ##########################################################################
 #
 # Automatic dependency generation
@@ -640,7 +684,6 @@ lcov.info: $(gcda_files)
 	$(GCOV) -b -f -p -o . $(GCOVFLAGS) $*.c >$*.c.gcov.out
 
 coverage: $(gcda_files:.gcda=.c.gcov) lcov.info
-	$(if $(SUBDIRS),for dir in $(SUBDIRS); do $(MAKE) -C $$dir coverage || exit; done)
 
 .PHONY: coverage-html
 coverage-html: coverage

src/Makefile.shlib

@@ -22,6 +22,7 @@
 # OBJS                  List of object files to include in library
 # SHLIB_LINK            If shared library relies on other libraries,
 #                       additional stuff to put in its link command
+# SHLIB_PREREQS         Order-only prerequisites for library build target
 # SHLIB_EXPORTS         (optional) Name of file containing list of symbols to
 #                       export, in the format "function_name  number"
 #
@@ -340,7 +341,7 @@ all-static-lib: $(stlib)
 all-shared-lib: $(shlib)
 
 ifndef haslibarule
-$(stlib): $(OBJS)
+$(stlib): $(OBJS) | $(SHLIB_PREREQS)
 	$(LINK.static) $@ $^
 	$(RANLIB) $@
 endif #haslibarule
@@ -351,7 +352,7 @@ ifeq (,$(filter cygwin win32,$(PORTNAME)))
 ifneq ($(PORTNAME), aix)
 
 # Normal case
-$(shlib): $(OBJS)
+$(shlib): $(OBJS) | $(SHLIB_PREREQS)
 	$(LINK.shared) -o $@ $(OBJS) $(LDFLAGS) $(LDFLAGS_SL) $(SHLIB_LINK)
 ifdef shlib_major
 # If we're using major and minor versions, then make a symlink to major-version-only.
@@ -382,7 +383,7 @@ endif
 else # PORTNAME == aix
 
 # AIX case
-$(shlib) $(stlib): $(OBJS)
+$(shlib) $(stlib): $(OBJS) | $(SHLIB_PREREQS)
 	$(LINK.static) $(stlib) $^
 	$(RANLIB) $(stlib)
 	$(MKLDEXPORT) $(stlib) >$(exports_file)
@@ -408,10 +409,10 @@ else
 DLL_DEFFILE = lib$(NAME)dll.def
 endif
 
-$(shlib): $(OBJS) $(DLL_DEFFILE)
+$(shlib): $(OBJS) $(DLL_DEFFILE) | $(SHLIB_PREREQS)
 	$(DLLWRAP) -o $@ --dllname $(shlib) $(DLLWRAP_FLAGS) --def $(DLL_DEFFILE) $(OBJS) $(LDFLAGS) $(LDFLAGS_SL) $(SHLIB_LINK)
 
-$(stlib): $(shlib) $(DLL_DEFFILE)
+$(stlib): $(shlib) $(DLL_DEFFILE) | $(SHLIB_PREREQS)
 	$(DLLTOOL) --dllname $(shlib) $(DLLTOOL_LIBFLAGS) --def $(DLL_DEFFILE) --output-lib $@
 
 endif # PORTNAME == cygwin || PORTNAME == win32

src/backend/Makefile

@@ -122,7 +122,9 @@ submake-schemapg:
 
 .PHONY: submake-schemapg
 
-catalog/schemapg.h: submake-schemapg
+catalog/schemapg.h: | submake-schemapg
+
+$(top_builddir)/src/port/libpgport_srv.a: | submake-libpgport
 
 
 # The postgres.o target is needed by the rule in Makefile.global that

src/backend/common.mk

@@ -34,15 +34,13 @@ objfiles.txt: Makefile $(SUBDIROBJS) $(OBJS)
 expand_subsys = $(foreach file,$(1),$(if $(filter %/objfiles.txt,$(file)),$(patsubst ../../src/backend/%,%,$(addprefix $(top_builddir)/,$(shell cat $(file)))),$(file)))
 
 # Parallel make trickery
-$(SUBDIROBJS): $(SUBDIRS:%=%-recursive) ;
+$(SUBDIROBJS): | $(SUBDIRS:%=%-recursive) ;
 
 .PHONY: $(SUBDIRS:%=%-recursive)
 $(SUBDIRS:%=%-recursive):
 	$(MAKE) -C $(subst -recursive,,$@) all
 
+$(call recurse,clean)
 clean: clean-local
 clean-local:
-ifdef SUBDIRS
-	for dir in $(SUBDIRS); do $(MAKE) -C $$dir clean || exit; done
-endif
 	rm -f $(subsysfilename) $(OBJS)

src/backend/replication/libpqwalreceiver/Makefile

@@ -16,9 +16,10 @@ override CPPFLAGS := -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS)
 
 OBJS = libpqwalreceiver.o
 SHLIB_LINK = $(libpq)
+SHLIB_PREREQS = submake-libpq
 NAME = libpqwalreceiver
 
-all: submake-libpq all-shared-lib
+all: all-shared-lib
 
 include $(top_srcdir)/src/Makefile.shlib
 

src/backend/utils/mb/conversion_procs/Makefile

@@ -26,6 +26,8 @@ SUBDIRS = \
 	utf8_and_sjis utf8_and_win utf8_and_uhc \
 	utf8_and_euc2004 utf8_and_sjis2004 euc2004_sjis2004
 
+$(recurse)
+
 # conversion_name source_encoding destination_encoding function object
 CONVERSIONS = \
 		ascii_to_mic	SQL_ASCII MULE_INTERNAL ascii_to_mic ascii_and_mic \
@@ -162,7 +164,6 @@ CONVERSIONS = \
 		shift_jis_2004_to_euc_jis_2004 SHIFT_JIS_2004 EUC_JIS_2004 shift_jis_2004_to_euc_jis_2004 euc2004_sjis2004
 
 all: $(SQLSCRIPT)
-	@for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@ || exit; done
 
 $(SQLSCRIPT): Makefile
 ifeq ($(enable_shared), yes)
@@ -205,15 +206,12 @@ $(REGRESSION_SCRIPT): Makefile
 
 install: $(SQLSCRIPT) installdirs
 	$(INSTALL_DATA) $(SQLSCRIPT) '$(DESTDIR)$(datadir)'
-	@for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@ || exit; done
 
 installdirs:
 	$(MKDIR_P) '$(DESTDIR)$(datadir)' '$(DESTDIR)$(pkglibdir)'
 
 uninstall:
 	rm -f '$(DESTDIR)$(datadir)/$(SQLSCRIPT)'
-	@for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@ || exit; done
 
 clean distclean maintainer-clean:
 	rm -f $(SQLSCRIPT)
-	@for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@ || exit; done