Support OpenSSL 1.1.0.

Changes needed to build at all:

- Check for SSL_new in configure, now that SSL_library_init is a macro.
- Do not access struct members directly. This includes some new code in
  pgcrypto, to use the resource owner mechanism to ensure that we don't
  leak OpenSSL handles, now that we can't embed them in other structs
  anymore.
- RAND_SSLeay() -> RAND_OpenSSL()

Changes that were needed to silence deprecation warnings, but were not
strictly necessary:

- RAND_pseudo_bytes() -> RAND_bytes().
- SSL_library_init() and OpenSSL_config() -> OPENSSL_init_ssl()
- ASN1_STRING_data() -> ASN1_STRING_get0_data()
- DH_generate_parameters() -> DH_generate_parameters()
- Locking callbacks are not needed with OpenSSL 1.1.0 anymore. (Good
  riddance!)

Also change references to SSLEAY_VERSION_NUMBER with OPENSSL_VERSION_NUMBER,
for the sake of consistency. OPENSSL_VERSION_NUMBER has existed since time
immemorial.

Fix SSL test suite to work with OpenSSL 1.1.0. CA certificates must have
the "CA:true" basic constraint extension now, or OpenSSL will refuse them.
Regenerate the test certificates with that. The "openssl" binary, used to
generate the certificates, is also now more picky, and throws an error
if an X509 extension is specified in "req_extensions", but that section
is empty.

Backpatch to 9.5 and 9.6, per popular demand. The file structure was
somewhat different in earlier branches, so I didn't bother to go further
than that. In back-branches, we still support OpenSSL 0.9.7 and above.
OpenSSL 0.9.6 should still work too, but I didn't test it. In master, we
only support 0.9.8 and above.

Patch by Andreas Karlsson, with additional changes by me.

Discussion: <20160627151604.GD1051@msg.df7cb.de>

Author: hlinnaka

configure

@@ -9538,9 +9538,9 @@ else
   as_fn_error $? "library 'crypto' is required for OpenSSL" "$LINENO" 5
 fi
 
-     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_library_init in -lssl" >&5
-$as_echo_n "checking for SSL_library_init in -lssl... " >&6; }
-if ${ac_cv_lib_ssl_SSL_library_init+:} false; then :
+     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_new in -lssl" >&5
+$as_echo_n "checking for SSL_new in -lssl... " >&6; }
+if ${ac_cv_lib_ssl_SSL_new+:} false; then :
   $as_echo_n "(cached) " >&6
 else
   ac_check_lib_save_LIBS=$LIBS
@@ -9554,27 +9554,27 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 #ifdef __cplusplus
 extern "C"
 #endif
-char SSL_library_init ();
+char SSL_new ();
 int
 main ()
 {
-return SSL_library_init ();
+return SSL_new ();
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_lib_ssl_SSL_library_init=yes
+  ac_cv_lib_ssl_SSL_new=yes
 else
-  ac_cv_lib_ssl_SSL_library_init=no
+  ac_cv_lib_ssl_SSL_new=no
 fi
 rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext conftest.$ac_ext
 LIBS=$ac_check_lib_save_LIBS
 fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_library_init" >&5
-$as_echo "$ac_cv_lib_ssl_SSL_library_init" >&6; }
-if test "x$ac_cv_lib_ssl_SSL_library_init" = xyes; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_new" >&5
+$as_echo "$ac_cv_lib_ssl_SSL_new" >&6; }
+if test "x$ac_cv_lib_ssl_SSL_new" = xyes; then :
   cat >>confdefs.h <<_ACEOF
 #define HAVE_LIBSSL 1
 _ACEOF
@@ -9644,9 +9644,9 @@ else
   as_fn_error $? "library 'eay32' or 'crypto' is required for OpenSSL" "$LINENO" 5
 fi
 
-     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing SSL_library_init" >&5
-$as_echo_n "checking for library containing SSL_library_init... " >&6; }
-if ${ac_cv_search_SSL_library_init+:} false; then :
+     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing SSL_new" >&5
+$as_echo_n "checking for library containing SSL_new... " >&6; }
+if ${ac_cv_search_SSL_new+:} false; then :
   $as_echo_n "(cached) " >&6
 else
   ac_func_search_save_LIBS=$LIBS
@@ -9659,11 +9659,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 #ifdef __cplusplus
 extern "C"
 #endif
-char SSL_library_init ();
+char SSL_new ();
 int
 main ()
 {
-return SSL_library_init ();
+return SSL_new ();
   ;
   return 0;
 }
@@ -9676,25 +9676,25 @@ for ac_lib in '' ssleay32 ssl; do
     LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
   fi
   if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_search_SSL_library_init=$ac_res
+  ac_cv_search_SSL_new=$ac_res
 fi
 rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext
-  if ${ac_cv_search_SSL_library_init+:} false; then :
+  if ${ac_cv_search_SSL_new+:} false; then :
   break
 fi
 done
-if ${ac_cv_search_SSL_library_init+:} false; then :
+if ${ac_cv_search_SSL_new+:} false; then :
 
 else
-  ac_cv_search_SSL_library_init=no
+  ac_cv_search_SSL_new=no
 fi
 rm conftest.$ac_ext
 LIBS=$ac_func_search_save_LIBS
 fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_SSL_library_init" >&5
-$as_echo "$ac_cv_search_SSL_library_init" >&6; }
-ac_res=$ac_cv_search_SSL_library_init
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_SSL_new" >&5
+$as_echo "$ac_cv_search_SSL_new" >&6; }
+ac_res=$ac_cv_search_SSL_new
 if test "$ac_res" != no; then :
   test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
 

configure.in

@@ -1112,10 +1112,10 @@ if test "$with_openssl" = yes ; then
   dnl Order matters!
   if test "$PORTNAME" != "win32"; then
      AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, [], [AC_MSG_ERROR([library 'crypto' is required for OpenSSL])])
-     AC_CHECK_LIB(ssl,    SSL_library_init, [], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])])
+     AC_CHECK_LIB(ssl,    SSL_new, [], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])])
   else
      AC_SEARCH_LIBS(CRYPTO_new_ex_data, eay32 crypto, [], [AC_MSG_ERROR([library 'eay32' or 'crypto' is required for OpenSSL])])
-     AC_SEARCH_LIBS(SSL_library_init, ssleay32 ssl, [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])])
+     AC_SEARCH_LIBS(SSL_new, ssleay32 ssl, [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])])
   fi
   AC_CHECK_FUNCS([SSL_get_current_compression])
 fi

contrib/pgcrypto/internal.c

@@ -620,15 +620,6 @@ px_find_cipher(const char *name, PX_Cipher **res)
  * Randomness provider
  */
 
-/*
- * Use always strong randomness.
- */
-int
-px_get_pseudo_random_bytes(uint8 *dst, unsigned count)
-{
-	return px_get_random_bytes(dst, count);
-}
-
 static time_t seed_time = 0;
 static time_t check_time = 0;
 

contrib/pgcrypto/openssl.c

@@ -40,6 +40,9 @@
 #include <openssl/rand.h>
 #include <openssl/err.h>
 
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+
 /*
  * Max lengths we might want to handle.
  */
@@ -199,60 +202,113 @@ compat_find_digest(const char *name, PX_MD **res)
  * Hashes
  */
 
+/*
+ * To make sure we don't leak OpenSSL handles on abort, we keep OSSLDigest
+ * objects in a linked list, allocated in TopMemoryContext. We use the
+ * ResourceOwner mechanism to free them on abort.
+ */
 typedef struct OSSLDigest
 {
 	const EVP_MD *algo;
-	EVP_MD_CTX	ctx;
+	EVP_MD_CTX *ctx;
+
+	ResourceOwner owner;
+	struct OSSLDigest *next;
+	struct OSSLDigest *prev;
 } OSSLDigest;
 
+static OSSLDigest *open_digests = NULL;
+static bool resowner_callback_registered = false;
+
+static void
+free_openssldigest(OSSLDigest *digest)
+{
+	EVP_MD_CTX_destroy(digest->ctx);
+	if (digest->prev)
+		digest->prev->next = digest->next;
+	else
+		open_digests = digest->next;
+	if (digest->next)
+		digest->next->prev = digest->prev;
+	pfree(digest);
+}
+
+/*
+ * Close any open OpenSSL handles on abort.
+ */
+static void
+digest_free_callback(ResourceReleasePhase phase,
+					 bool isCommit,
+					 bool isTopLevel,
+					 void *arg)
+{
+	OSSLDigest *curr;
+	OSSLDigest *next;
+
+	if (phase != RESOURCE_RELEASE_AFTER_LOCKS)
+		return;
+
+	next = open_digests;
+	while (next)
+	{
+		curr = next;
+		next = curr->next;
+
+		if (curr->owner == CurrentResourceOwner)
+		{
+			if (isCommit)
+				elog(WARNING, "pgcrypto digest reference leak: digest %p still referenced", curr);
+			free_openssldigest(curr);
+		}
+	}
+}
+
 static unsigned
 digest_result_size(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	return EVP_MD_CTX_size(&digest->ctx);
+	return EVP_MD_CTX_size(digest->ctx);
 }
 
 static unsigned
 digest_block_size(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	return EVP_MD_CTX_block_size(&digest->ctx);
+	return EVP_MD_CTX_block_size(digest->ctx);
 }
 
 static void
 digest_reset(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL);
+	EVP_DigestInit_ex(digest->ctx, digest->algo, NULL);
 }
 
 static void
 digest_update(PX_MD *h, const uint8 *data, unsigned dlen)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_DigestUpdate(&digest->ctx, data, dlen);
+	EVP_DigestUpdate(digest->ctx, data, dlen);
 }
 
 static void
 digest_finish(PX_MD *h, uint8 *dst)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_DigestFinal_ex(&digest->ctx, dst, NULL);
+	EVP_DigestFinal_ex(digest->ctx, dst, NULL);
 }
 
 static void
 digest_free(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_MD_CTX_cleanup(&digest->ctx);
-
-	px_free(digest);
+	free_openssldigest(digest);
 	px_free(h);
 }
 
@@ -264,6 +320,7 @@ int
 px_find_digest(const char *name, PX_MD **res)
 {
 	const EVP_MD *md;
+	EVP_MD_CTX *ctx;
 	PX_MD	   *h;
 	OSSLDigest *digest;
 
@@ -273,17 +330,43 @@ px_find_digest(const char *name, PX_MD **res)
 		OpenSSL_add_all_algorithms();
 	}
 
+	if (!resowner_callback_registered)
+	{
+		RegisterResourceReleaseCallback(digest_free_callback, NULL);
+		resowner_callback_registered = true;
+	}
+
 	md = EVP_get_digestbyname(name);
 	if (md == NULL)
 		return compat_find_digest(name, res);
 
-	digest = px_alloc(sizeof(*digest));
-	digest->algo = md;
+	/*
+	 * Create an OSSLDigest object, an OpenSSL MD object, and a PX_MD object.
+	 * The order is crucial, to make sure we don't leak anything on
+	 * out-of-memory or other error.
+	 */
+	digest = MemoryContextAlloc(TopMemoryContext, sizeof(*digest));
 
-	EVP_MD_CTX_init(&digest->ctx);
-	if (EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL) == 0)
+	ctx = EVP_MD_CTX_create();
+	if (!ctx)
+	{
+		pfree(digest);
 		return -1;
+	}
+	if (EVP_DigestInit_ex(ctx, md, NULL) == 0)
+	{
+		pfree(digest);
+		return -1;
+	}
+
+	digest->algo = md;
+	digest->ctx = ctx;
+	digest->owner = CurrentResourceOwner;
+	digest->next = open_digests;
+	digest->prev = NULL;
+	open_digests = digest;
 
+	/* The PX_MD object is allocated in the current memory context. */
 	h = px_alloc(sizeof(*h));
 	h->result_size = digest_result_size;
 	h->block_size = digest_block_size;
@@ -979,6 +1062,10 @@ px_find_cipher(const char *name, PX_Cipher **res)
 
 static int	openssl_random_init = 0;
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define RAND_OpenSSL RAND_SSLeay
+#endif
+
 /*
  * OpenSSL random should re-feeded occasionally. From /dev/urandom
  * preferably.
@@ -987,7 +1074,7 @@ static void
 init_openssl_rand(void)
 {
 	if (RAND_get_rand_method() == NULL)
-		RAND_set_rand_method(RAND_SSLeay());
+		RAND_set_rand_method(RAND_OpenSSL());
 	openssl_random_init = 1;
 }
 
@@ -1006,21 +1093,6 @@ px_get_random_bytes(uint8 *dst, unsigned count)
 	return PXE_OSSL_RAND_ERROR;
 }
 
-int
-px_get_pseudo_random_bytes(uint8 *dst, unsigned count)
-{
-	int			res;
-
-	if (!openssl_random_init)
-		init_openssl_rand();
-
-	res = RAND_pseudo_bytes(dst, count);
-	if (res == 0 || res == 1)
-		return count;
-
-	return PXE_OSSL_RAND_ERROR;
-}
-
 int
 px_add_entropy(const uint8 *data, unsigned count)
 {

contrib/pgcrypto/pgcrypto.c

@@ -454,7 +454,7 @@ pg_random_uuid(PG_FUNCTION_ARGS)
 	int			err;
 
 	/* generate random bits */
-	err = px_get_pseudo_random_bytes(buf, UUID_LEN);
+	err = px_get_random_bytes(buf, UUID_LEN);
 	if (err < 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION),

contrib/pgcrypto/pgp-s2k.c

@@ -233,13 +233,13 @@ pgp_s2k_fill(PGP_S2K *s2k, int mode, int digest_algo, int count)
 		case PGP_S2K_SIMPLE:
 			break;
 		case PGP_S2K_SALTED:
-			res = px_get_pseudo_random_bytes(s2k->salt, PGP_S2K_SALT);
+			res = px_get_random_bytes(s2k->salt, PGP_S2K_SALT);
 			break;
 		case PGP_S2K_ISALTED:
-			res = px_get_pseudo_random_bytes(s2k->salt, PGP_S2K_SALT);
+			res = px_get_random_bytes(s2k->salt, PGP_S2K_SALT);
 			if (res < 0)
 				break;
-			res = px_get_pseudo_random_bytes(&tmp, 1);
+			res = px_get_random_bytes(&tmp, 1);
 			if (res < 0)
 				break;
 			s2k->iter = decide_s2k_iter(tmp, count);