Design to extend the varattrib_4b/varatt_external format for support of multiple TOAST compression algorithms.

Author: nikhilveldanda

contrib/amcheck/verify_heapam.c

@@ -1786,7 +1786,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
 		bool		valid = false;
 
 		/* Compressed attributes should have a valid compression method */
-		cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+		cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
 		switch (cmid)
 		{
 				/* List of all valid compression method IDs */

src/backend/access/brin/brin_tuple.c

@@ -223,6 +223,7 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
 			{
 				Datum		cvalue;
 				char		compression;
+				CompressionInfo cmp;
 				Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
 													  keyno);
 
@@ -237,7 +238,8 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
 				else
 					compression = InvalidCompressionMethod;
 
-				cvalue = toast_compress_datum(value, compression);
+				cmp = setup_cmp_info(compression, att);
+				cvalue = toast_compress_datum(value, cmp);
 
 				if (DatumGetPointer(cvalue) != NULL)
 				{

src/backend/access/common/detoast.c

@@ -478,7 +478,7 @@ toast_decompress_datum(struct varlena *attr)
 	 * Fetch the compression method id stored in the compression header and
 	 * decompress the data using the appropriate decompression routine.
 	 */
-	cmid = TOAST_COMPRESS_METHOD(attr);
+	cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 	switch (cmid)
 	{
 		case TOAST_PGLZ_COMPRESSION_ID:
@@ -514,14 +514,14 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
 	 * have been seen to give wrong results if passed an output size that is
 	 * more than the data's true decompressed size.
 	 */
-	if ((uint32) slicelength >= TOAST_COMPRESS_EXTSIZE(attr))
+	if ((uint32) slicelength >= VARDATA_COMPRESSED_GET_EXTSIZE(attr))
 		return toast_decompress_datum(attr);
 
 	/*
 	 * Fetch the compression method id stored in the compression header and
 	 * decompress the data slice using the appropriate decompression routine.
 	 */
-	cmid = TOAST_COMPRESS_METHOD(attr);
+	cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 	switch (cmid)
 	{
 		case TOAST_PGLZ_COMPRESSION_ID:

src/backend/access/common/indextuple.c

@@ -123,9 +123,10 @@ index_form_tuple_context(TupleDesc tupleDescriptor,
 			 att->attstorage == TYPSTORAGE_MAIN))
 		{
 			Datum		cvalue;
+			CompressionInfo cmp;
 
-			cvalue = toast_compress_datum(untoasted_values[i],
-										  att->attcompression);
+			cmp = setup_cmp_info(att->attcompression, att);
+			cvalue = toast_compress_datum(untoasted_values[i], cmp);
 
 			if (DatumGetPointer(cvalue) != NULL)
 			{

src/backend/access/common/toast_compression.c

@@ -21,6 +21,7 @@
 #include "access/toast_compression.h"
 #include "common/pg_lzcompress.h"
 #include "varatt.h"
+#include "utils/attoptcache.h"
 
 /* GUC */
 int			default_toast_compression = TOAST_PGLZ_COMPRESSION;
@@ -314,3 +315,28 @@ GetCompressionMethodName(char method)
 			return NULL;		/* keep compiler quiet */
 	}
 }
+
+CompressionInfo
+setup_cmp_info(char cmethod, Form_pg_attribute att)
+{
+	CompressionInfo info;
+
+	/* initialize from the attribute’s default settings */
+	info.cmethod = cmethod;
+	info.meta = false;
+
+	/* If the compression method is not valid, use the current default */
+	if (!CompressionMethodIsValid(cmethod))
+		info.cmethod = default_toast_compression;
+
+	switch (info.cmethod)
+	{
+		case TOAST_PGLZ_COMPRESSION:
+		case TOAST_LZ4_COMPRESSION:
+			break;
+		default:
+			elog(ERROR, "invalid compression method %c", info.cmethod);
+	}
+
+	return info;
+}

src/backend/access/common/toast_internals.c

@@ -43,7 +43,7 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
  * ----------
  */
 Datum
-toast_compress_datum(Datum value, char cmethod)
+toast_compress_datum(Datum value, CompressionInfo cmp)
 {
 	struct varlena *tmp = NULL;
 	int32		valsize;
@@ -54,14 +54,10 @@ toast_compress_datum(Datum value, char cmethod)
 
 	valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
 
-	/* If the compression method is not valid, use the current default */
-	if (!CompressionMethodIsValid(cmethod))
-		cmethod = default_toast_compression;
-
 	/*
 	 * Call appropriate compression routine for the compression method.
 	 */
-	switch (cmethod)
+	switch (cmp.cmethod)
 	{
 		case TOAST_PGLZ_COMPRESSION:
 			tmp = pglz_compress_datum((const struct varlena *) value);
@@ -72,7 +68,7 @@ toast_compress_datum(Datum value, char cmethod)
 			cmid = TOAST_LZ4_COMPRESSION_ID;
 			break;
 		default:
-			elog(ERROR, "invalid compression method %c", cmethod);
+			elog(ERROR, "invalid compression method %c", cmp.cmethod);
 	}
 
 	if (tmp == NULL)
@@ -90,9 +86,11 @@ toast_compress_datum(Datum value, char cmethod)
 	 */
 	if (VARSIZE(tmp) < valsize - 2)
 	{
+		bool		meta = cmp.meta;
+
 		/* successful compression */
 		Assert(cmid != TOAST_INVALID_COMPRESSION_ID);
-		TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid);
+		TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(tmp, valsize, cmid, meta);
 		return PointerGetDatum(tmp);
 	}
 	else
@@ -143,6 +141,7 @@ toast_save_datum(Relation rel, Datum value,
 	Pointer		dval = DatumGetPointer(value);
 	int			num_indexes;
 	int			validIndex;
+	ToastCompressionId cm = TOAST_INVALID_COMPRESSION_ID;
 
 	Assert(!VARATT_IS_EXTERNAL(value));
 
@@ -179,14 +178,18 @@ toast_save_datum(Relation rel, Datum value,
 	}
 	else if (VARATT_IS_COMPRESSED(dval))
 	{
+		bool		meta;
+
 		data_p = VARDATA(dval);
 		data_todo = VARSIZE(dval) - VARHDRSZ;
 		/* rawsize in a compressed datum is just the size of the payload */
 		toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
-
+		cm = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval);
+		meta = TOAST_CMPID_EXTENDED(cm) ?
+			VARATT_4BCE_HAS_META(((varattrib_4b *) (dval))->va_compressed_ext.va_ecinfo) :
+			false;
 		/* set external size and compression method */
-		VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
-													 VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval));
+		VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo, cm, meta);
 		/* Assert that the numbers look like it's compressed */
 		Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
 	}
@@ -368,9 +371,9 @@ toast_save_datum(Relation rel, Datum value,
 	/*
 	 * Create the TOAST pointer value that we'll return
 	 */
-	result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
+	result = (struct varlena *) palloc(TOAST_CMPID_EXTENDED(cm) ? TOAST_POINTER_EXT_SIZE : TOAST_POINTER_NOEXT_SIZE);
 	SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK);
-	memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+	memcpy(VARDATA_EXTERNAL(result), &toast_pointer, TOAST_CMPID_EXTENDED(cm) ? TOAST_POINTER_EXT_SIZE - VARHDRSZ_EXTERNAL : TOAST_POINTER_NOEXT_SIZE - VARHDRSZ_EXTERNAL);
 
 	return PointerGetDatum(result);
 }

src/backend/access/table/toast_helper.c

@@ -171,7 +171,7 @@ toast_tuple_init(ToastTupleContext *ttc)
  * The column must have attstorage EXTERNAL or EXTENDED if check_main is
  * false, and must have attstorage MAIN if check_main is true.
  *
- * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
+ * The column must have a minimum size of MAXALIGN(TOAST_POINTER_NOEXT_SIZE);
  * if not, no benefit is to be expected by compressing it.
  *
  * The return value is the index of the biggest suitable column, or
@@ -184,7 +184,7 @@ toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
 	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
 	int			numAttrs = tupleDesc->natts;
 	int			biggest_attno = -1;
-	int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
+	int32		biggest_size = MAXALIGN(TOAST_POINTER_NOEXT_SIZE);
 	int32		skip_colflags = TOASTCOL_IGNORE;
 	int			i;
 
@@ -229,8 +229,10 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
 	Datum	   *value = &ttc->ttc_values[attribute];
 	Datum		new_value;
 	ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
+	Form_pg_attribute att = TupleDescAttr(ttc->ttc_rel->rd_att, attribute);
+	CompressionInfo cmp = setup_cmp_info(attr->tai_compression, att);
 
-	new_value = toast_compress_datum(*value, attr->tai_compression);
+	new_value = toast_compress_datum(*value, cmp);
 
 	if (DatumGetPointer(new_value) != NULL)
 	{

src/include/access/detoast.h

@@ -23,12 +23,13 @@
 do { \
 	varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
 	Assert(VARATT_IS_EXTERNAL(attre)); \
-	Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
-	memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
+	memset(&(toast_pointer), 0, sizeof(toast_pointer)); \
+	memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), VARSIZE_EXTERNAL(attre) - VARHDRSZ_EXTERNAL); \
 } while (0)
 
 /* Size of an EXTERNAL datum that contains a standard TOAST pointer */
-#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
+#define TOAST_POINTER_NOEXT_SIZE (VARHDRSZ_EXTERNAL + offsetof(varatt_external, extended))
+#define TOAST_POINTER_EXT_SIZE (TOAST_POINTER_NOEXT_SIZE + MEMBER_SIZE(varatt_external, extended.cmp))
 
 /* Size of an EXTERNAL datum that contains an indirection pointer */
 #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))

src/include/access/toast_compression.h

@@ -13,6 +13,8 @@
 #ifndef TOAST_COMPRESSION_H
 #define TOAST_COMPRESSION_H
 
+#include "catalog/pg_attribute.h"
+
 /*
  * GUC support.
  *
@@ -22,25 +24,19 @@
  */
 extern PGDLLIMPORT int default_toast_compression;
 
-/*
- * Built-in compression method ID.  The toast compression header will store
- * this in the first 2 bits of the raw length.  These built-in compression
- * method IDs are directly mapped to the built-in compression methods.
- *
- * Don't use these values for anything other than understanding the meaning
- * of the raw bits from a varlena; in particular, if the goal is to identify
- * a compression method, use the constants TOAST_PGLZ_COMPRESSION, etc.
- * below. We might someday support more than 4 compression methods, but
- * we can never have more than 4 values in this enum, because there are
- * only 2 bits available in the places where this is stored.
- */
 typedef enum ToastCompressionId
 {
 	TOAST_PGLZ_COMPRESSION_ID = 0,
 	TOAST_LZ4_COMPRESSION_ID = 1,
 	TOAST_INVALID_COMPRESSION_ID = 2,
 } ToastCompressionId;
 
+typedef struct CompressionInfo
+{
+	char		cmethod;
+	bool		meta;
+} CompressionInfo;
+
 /*
  * Built-in compression methods.  pg_attribute will store these in the
  * attcompression column.  In attcompression, InvalidCompressionMethod
@@ -51,6 +47,7 @@ typedef enum ToastCompressionId
 #define InvalidCompressionMethod		'\0'
 
 #define CompressionMethodIsValid(cm)  ((cm) != InvalidCompressionMethod)
+#define TOAST_CMPID_EXTENDED(cmpid)	(!(cmpid == TOAST_PGLZ_COMPRESSION_ID || cmpid == TOAST_LZ4_COMPRESSION_ID ||cmpid == TOAST_INVALID_COMPRESSION_ID))
 
 
 /* pglz compression/decompression routines */
@@ -69,5 +66,6 @@ extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
 extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
 extern char CompressionNameToMethod(const char *compression);
 extern const char *GetCompressionMethodName(char method);
+extern CompressionInfo setup_cmp_info(char cmethod, Form_pg_attribute att);
 
 #endif							/* TOAST_COMPRESSION_H */

src/include/access/toast_internals.h

@@ -17,35 +17,31 @@
 #include "utils/relcache.h"
 #include "utils/snapshot.h"
 
-/*
- *	The information at the start of the compressed toast data.
- */
-typedef struct toast_compress_header
-{
-	int32		vl_len_;		/* varlena header (do not touch directly!) */
-	uint32		tcinfo;			/* 2 bits for compression method and 30 bits
-								 * external size; see va_extinfo */
-} toast_compress_header;
-
 /*
  * Utilities for manipulation of header information for compressed
  * toast entries.
  */
-#define TOAST_COMPRESS_EXTSIZE(ptr) \
-	(((toast_compress_header *) (ptr))->tcinfo & VARLENA_EXTSIZE_MASK)
-#define TOAST_COMPRESS_METHOD(ptr) \
-	(((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
-
-#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
-	do { \
-		Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
-		Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
-			   (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
-		((toast_compress_header *) (ptr))->tcinfo = \
-			(len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
+#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method, meta)			\
+	do {																				\
+		Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK);								\
+		Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID ||								\
+				(cm_method) == TOAST_LZ4_COMPRESSION_ID);								\
+		if (!TOAST_CMPID_EXTENDED((cm_method)))											\
+		{																				\
+			((varattrib_4b *)(ptr))->va_compressed.va_tcinfo =							\
+				((uint32)(len)) | ((uint32)(cm_method) << VARLENA_EXTSIZE_BITS);		\
+		}																				\
+		else																			\
+		{																				\
+			/* extended path: mark EXT flag in tcinfo */								\
+			((varattrib_4b *)(ptr))->va_compressed_ext.va_tcinfo =						\
+				((uint32)(len)) | ((uint32)(VARATT_4BCE_EXTFLAG) << VARLENA_EXTSIZE_BITS);	\
+			((varattrib_4b *)(ptr))->va_compressed_ext.va_ecinfo = 						\
+				VARATT_4BCE_ENCODE((meta), (cm_method));							\
+		}																				\
 	} while (0)
 
-extern Datum toast_compress_datum(Datum value, char cmethod);
+extern Datum toast_compress_datum(Datum value, CompressionInfo cmp);
 extern Oid	toast_get_valid_index(Oid toastoid, LOCKMODE lock);
 
 extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);