Fix typcache's failure to treat ranges as container types.

Like the similar logic for arrays and records, it's necessary to examine
the range's subtype to decide whether the range type can support hashing.
We can omit checking the subtype for btree-defined operations, though,
since range subtypes are required to have those operations.  (Possibly
that simplification for btree cases led us to overlook that it does
not apply for hash cases.)

This is only an issue if the subtype lacks hash support, which is not
true of any built-in range type, but it's easy to demonstrate a problem
with a range type over, eg, money: you can get a "could not identify
a hash function" failure when the planner is misled into thinking that
hash join or aggregation would work.

This was born broken, so back-patch to all supported branches.

Author: tglsfdc

src/backend/utils/cache/typcache.c

@@ -168,6 +168,8 @@ static void cache_array_element_properties(TypeCacheEntry *typentry);
 static bool record_fields_have_equality(TypeCacheEntry *typentry);
 static bool record_fields_have_compare(TypeCacheEntry *typentry);
 static void cache_record_field_properties(TypeCacheEntry *typentry);
+static bool range_element_has_hashing(TypeCacheEntry *typentry);
+static void cache_range_element_properties(TypeCacheEntry *typentry);
 static void TypeCacheRelCallback(Datum arg, Oid relid);
 static void TypeCacheOpcCallback(Datum arg, int cacheid, uint32 hashvalue);
 static void TypeCacheConstrCallback(Datum arg, int cacheid, uint32 hashvalue);
@@ -485,6 +487,13 @@ lookup_type_cache(Oid type_id, int flags)
 			!array_element_has_hashing(typentry))
 			hash_proc = InvalidOid;
 
+		/*
+		 * Likewise for hash_range.
+		 */
+		if (hash_proc == F_HASH_RANGE &&
+			!range_element_has_hashing(typentry))
+			hash_proc = InvalidOid;
+
 		/* Force update of hash_proc_finfo only if we're changing state */
 		if (typentry->hash_proc != hash_proc)
 			typentry->hash_proc_finfo.fn_oid = InvalidOid;
@@ -1129,6 +1138,10 @@ cache_array_element_properties(TypeCacheEntry *typentry)
 	typentry->flags |= TCFLAGS_CHECKED_ELEM_PROPERTIES;
 }
 
+/*
+ * Likewise, some helper functions for composite types.
+ */
+
 static bool
 record_fields_have_equality(TypeCacheEntry *typentry)
 {
@@ -1199,6 +1212,43 @@ cache_record_field_properties(TypeCacheEntry *typentry)
 	typentry->flags |= TCFLAGS_CHECKED_FIELD_PROPERTIES;
 }
 
+/*
+ * Likewise, some helper functions for range types.
+ *
+ * We can borrow the flag bits for array element properties to use for range
+ * element properties, since those flag bits otherwise have no use in a
+ * range type's typcache entry.
+ */
+
+static bool
+range_element_has_hashing(TypeCacheEntry *typentry)
+{
+	if (!(typentry->flags & TCFLAGS_CHECKED_ELEM_PROPERTIES))
+		cache_range_element_properties(typentry);
+	return (typentry->flags & TCFLAGS_HAVE_ELEM_HASHING) != 0;
+}
+
+static void
+cache_range_element_properties(TypeCacheEntry *typentry)
+{
+	/* load up subtype link if we didn't already */
+	if (typentry->rngelemtype == NULL &&
+		typentry->typtype == TYPTYPE_RANGE)
+		load_rangetype_info(typentry);
+
+	if (typentry->rngelemtype != NULL)
+	{
+		TypeCacheEntry *elementry;
+
+		/* might need to calculate subtype's hash function properties */
+		elementry = lookup_type_cache(typentry->rngelemtype->type_id,
+									  TYPECACHE_HASH_PROC);
+		if (OidIsValid(elementry->hash_proc))
+			typentry->flags |= TCFLAGS_HAVE_ELEM_HASHING;
+	}
+	typentry->flags |= TCFLAGS_CHECKED_ELEM_PROPERTIES;
+}
+
 
 /*
  * lookup_rowtype_tupdesc_internal --- internal routine to lookup a rowtype

src/test/regress/expected/rangetypes.out

@@ -1354,6 +1354,18 @@ select *, row_to_json(upper(t)) as u from
 drop type two_ints cascade;
 NOTICE:  drop cascades to type two_ints_range
 --
+-- Check behavior when subtype lacks a hash function
+--
+create type cashrange as range (subtype = money);
+set enable_sort = off;  -- try to make it pick a hash setop implementation
+select '(2,5)'::cashrange except select '(5,6)'::cashrange;
+   cashrange   
+---------------
+ ($2.00,$5.00)
+(1 row)
+
+reset enable_sort;
+--
 -- OUT/INOUT/TABLE functions
 --
 create function outparam_succeed(i anyrange, out r anyrange, out t text)

src/test/regress/sql/rangetypes.sql

@@ -461,6 +461,18 @@ select *, row_to_json(upper(t)) as u from
 
 drop type two_ints cascade;
 
+--
+-- Check behavior when subtype lacks a hash function
+--
+
+create type cashrange as range (subtype = money);
+
+set enable_sort = off;  -- try to make it pick a hash setop implementation
+
+select '(2,5)'::cashrange except select '(5,6)'::cashrange;
+
+reset enable_sort;
+
 --
 -- OUT/INOUT/TABLE functions
 --