Make catalog cache hash tables resizeable.

If the hash table backing a catalog cache becomes too full (fillfactor > 2),
enlarge it. A new buckets array, double the size of the old, is allocated,
and all entries in the old hash are moved to the right bucket in the new
hash.

This has two benefits. First, cache lookups don't get so expensive when
there are lots of entries in a cache, like if you access hundreds of
thousands of tables. Second, we can make the (initial) sizes of the caches
much smaller, which saves memory.

This patch dials down the initial sizes of the catcaches. The new sizes are
chosen so that a backend that only runs a few basic queries still won't need
to enlarge any of them.

Author: hlinnaka

src/backend/utils/cache/catcache.c

@@ -734,9 +734,8 @@ InitCatCache(int id,
 	int			i;
 
 	/*
-	 * nbuckets is the number of hash buckets to use in this catcache.
-	 * Currently we just use a hard-wired estimate of an appropriate size for
-	 * each cache; maybe later make them dynamically resizable?
+	 * nbuckets is the initial number of hash buckets to use in this catcache.
+	 * It will be enlarged later if it becomes too full.
 	 *
 	 * nbuckets must be a power of two.  We check this via Assert rather than
 	 * a full runtime check because the values will be coming from constant
@@ -775,7 +774,8 @@ InitCatCache(int id,
 	 *
 	 * Note: we rely on zeroing to initialize all the dlist headers correctly
 	 */
-	cp = (CatCache *) palloc0(sizeof(CatCache) + nbuckets * sizeof(dlist_head));
+	cp = (CatCache *) palloc0(sizeof(CatCache));
+	cp->cc_bucket = palloc0(nbuckets * sizeof(dlist_head));
 
 	/*
 	 * initialize the cache's relation information for the relation
@@ -813,6 +813,43 @@ InitCatCache(int id,
 	return cp;
 }
 
+/*
+ * Enlarge a catcache, doubling the number of buckets.
+ */
+static void
+RehashCatCache(CatCache *cp)
+{
+	dlist_head *newbucket;
+	int			newnbuckets;
+	int			i;
+
+	elog(DEBUG1, "rehashing catalog cache id %d for %s; %d tups, %d buckets",
+		 cp->id, cp->cc_relname, cp->cc_ntup, cp->cc_nbuckets);
+
+	/* Allocate a new, larger, hash table. */
+	newnbuckets = cp->cc_nbuckets * 2;
+	newbucket = (dlist_head *) MemoryContextAllocZero(CacheMemoryContext, newnbuckets * sizeof(dlist_head));
+
+	/* Move all entries from old hash table to new. */
+	for (i = 0; i < cp->cc_nbuckets; i++)
+	{
+		dlist_mutable_iter iter;
+		dlist_foreach_modify(iter, &cp->cc_bucket[i])
+		{
+			CatCTup	   *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+			int			hashIndex = HASH_INDEX(ct->hash_value, newnbuckets);
+
+			dlist_delete(iter.cur);
+			dlist_push_head(&newbucket[hashIndex], &ct->cache_elem);
+		}
+	}
+
+	/* Switch to the new array. */
+	pfree(cp->cc_bucket);
+	cp->cc_nbuckets = newnbuckets;
+	cp->cc_bucket = newbucket;
+}
+
 /*
  *		CatalogCacheInitializeCache
  *
@@ -1684,6 +1721,13 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp,
 	cache->cc_ntup++;
 	CacheHdr->ch_ntup++;
 
+	/*
+	 * If the hash table has become too full, enlarge the buckets array.
+	 * Quite arbitrarily, we enlarge when fill factor > 2.
+	 */
+	if (cache->cc_ntup > cache->cc_nbuckets * 2)
+		RehashCatCache(cache);
+
 	return ct;
 }