Permit dump/reload of not-too-large >1GB tuples

Our documentation states that our maximum field size is 1 GB, and that
our maximum row size of 1.6 TB.  However, while this might be attainable
in theory with enough contortions, it is not workable in practice; for
starters, pg_dump fails to dump tables containing rows larger than 1 GB,
even if individual columns are well below the limit; and even if one
does manage to manufacture a dump file containing a row that large, the
server refuses to load it anyway.

This commit enables dumping and reloading of such tuples, provided two
conditions are met:

1. no single column is larger than 1 GB (in output size -- for bytea
   this includes the formatting overhead)
2. the whole row is not larger than 2 GB

There are three related changes to enable this:

a. StringInfo's API now has two additional functions that allow creating
a string that grows beyond the typical 1GB limit (and "long" string).
ABI compatibility is maintained.  We still limit these strings to 2 GB,
though, for reasons explained below.

b. COPY now uses long StringInfos, so that pg_dump doesn't choke
trying to emit rows longer than 1GB.

c. heap_form_tuple now uses the MCXT_ALLOW_HUGE flag in its allocation
for the input tuple, which means that large tuples are accepted on
input.  Note that at this point we do not apply any further limit to the
input tuple size.

The main reason to limit to 2 GB is that the FE/BE protocol uses 32 bit
length words to describe each row; and because the documentation is
ambiguous on its signedness and libpq does consider it signed, we cannot
use the highest-order bit.  Additionally, the StringInfo API uses "int"
(which is 4 bytes wide in most platforms) in many places, so we'd need
to change that API too in order to improve, which has lots of fallout.

Backpatch to 9.5, which is the oldest that has
MemoryContextAllocExtended, a necessary piece of infrastructure.  We
could apply to 9.4 with very minimal additional effort, but any further
than that would require backpatching "huge" allocations too.

This is the largest set of changes we could find that can be
back-patched without breaking compatibility with existing systems.
Fixing a bigger set of problems (for example, dumping tuples bigger than
2GB, or dumping fields bigger than 1GB) would require changing the FE/BE
protocol and/or changing the StringInfo API in an ABI-incompatible way,
neither of which would be back-patchable.

Authors: Daniel Vérité, Álvaro Herrera
Reviewed by: Tomas Vondra
Discussion: https://postgr.es/m/20160229183023.GA286012@alvherre.pgsql

diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c
index 4363bc3d7e83d04ae6984f2d30b155d4e0c2575e..68eaa4bb41d7b79a6a2a6e706292b01d490007da 100644 (file)
--- a/src/backend/access/common/heaptuple.c
+++ b/src/backend/access/common/heaptuple.c
@@ -741,7 +741,9 @@ heap_form_tuple(TupleDesc tupleDescriptor,
     * Allocate and zero the space needed.  Note that the tuple body and
     * HeapTupleData management structure are allocated in one chunk.
     */
-   tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
+   tuple = MemoryContextAllocExtended(CurrentMemoryContext,
+                                      HEAPTUPLESIZE + len,
+                                      MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO);
    tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
 
    /*

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 96ed21aae93e3ba9ed0a14025ee9a2163b2dd710..c4a2c0fda921f7894549066588393f2ee5a17746 100644 (file)
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -400,7 +400,7 @@ ReceiveCopyBegin(CopyState cstate)
            pq_sendint(&buf, format, 2);        /* per-column formats */
        pq_endmessage(&buf);
        cstate->copy_dest = COPY_NEW_FE;
-       cstate->fe_msgbuf = makeStringInfo();
+       cstate->fe_msgbuf = makeLongStringInfo();
    }
    else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2)
    {
@@ -1865,7 +1865,7 @@ CopyTo(CopyState cstate)
    cstate->null_print_client = cstate->null_print;     /* default */
 
    /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
-   cstate->fe_msgbuf = makeStringInfo();
+   cstate->fe_msgbuf = makeLongStringInfo();
 
    /* Get info about the columns we need to process. */
    cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
@@ -2681,8 +2681,8 @@ BeginCopyFrom(Relation rel,
    cstate->cur_attval = NULL;
 
    /* Set up variables to avoid per-attribute overhead. */
-   initStringInfo(&cstate->attribute_buf);
-   initStringInfo(&cstate->line_buf);
+   initLongStringInfo(&cstate->attribute_buf);
+   initLongStringInfo(&cstate->line_buf);
    cstate->line_buf_converted = false;
    cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
    cstate->raw_buf_index = cstate->raw_buf_len = 0;

diff --git a/src/backend/lib/stringinfo.c b/src/backend/lib/stringinfo.c
index c725dde5793e1cd8771af083da899c1556759100..c3c1b41a049b4f41eaa30a7ef48997085fcae961 100644 (file)
--- a/src/backend/lib/stringinfo.c
+++ b/src/backend/lib/stringinfo.c
@@ -4,7 +4,8 @@
  *
  * StringInfo provides an indefinitely-extensible string data type.
  * It can be used to buffer either ordinary C strings (null-terminated text)
- * or arbitrary binary data.  All storage is allocated with palloc().
+ * or arbitrary binary data.  All storage is allocated with palloc() and
+ * friends.
  *
  * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
@@ -36,11 +37,29 @@ makeStringInfo(void)
    return res;
 }
 
+/*
+ * makeLongStringInfo
+ *
+ * Same as makeStringInfo, for larger strings.
+ */
+StringInfo
+makeLongStringInfo(void)
+{
+   StringInfo  res;
+
+   res = (StringInfo) palloc(sizeof(StringInfoData));
+
+   initLongStringInfo(res);
+
+   return res;
+}
+
+
 /*
  * initStringInfo
  *
  * Initialize a StringInfoData struct (with previously undefined contents)
- * to describe an empty string.
+ * to describe an empty string; don't enable long strings yet.
  */
 void
 initStringInfo(StringInfo str)
@@ -49,9 +68,22 @@ initStringInfo(StringInfo str)
 
    str->data = (char *) palloc(size);
    str->maxlen = size;
+   str->long_ok = false;
    resetStringInfo(str);
 }
 
+/*
+ * initLongStringInfo
+ *
+ * Same as initStringInfo, plus enable long strings.
+ */
+void
+initLongStringInfo(StringInfo str)
+{
+   initStringInfo(str);
+   str->long_ok = true;
+}
+
 /*
  * resetStringInfo
  *
@@ -142,7 +174,7 @@ appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
    /*
     * Return pvsnprintf's estimate of the space needed.  (Although this is
     * given as a size_t, we know it will fit in int because it's not more
-    * than MaxAllocSize.)
+    * than either MaxAllocSize or half an int's width.)
     */
    return (int) nprinted;
 }
@@ -244,7 +276,17 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
 void
 enlargeStringInfo(StringInfo str, int needed)
 {
-   int         newlen;
+   Size        newlen;
+   Size        limit;
+
+   /*
+    * Determine the upper size limit.  Because of overflow concerns outside
+    * of this module, we limit ourselves to 4-byte signed integer range,
+    * even for "long_ok" strings.
+    */
+   limit = str->long_ok ?
+       (((Size) 1) << (sizeof(int32) * 8 - 1)) - 1 :
+       MaxAllocSize;
 
    /*
     * Guard against out-of-range "needed" values.  Without this, we can get
@@ -252,7 +294,7 @@ enlargeStringInfo(StringInfo str, int needed)
     */
    if (needed < 0)             /* should not happen */
        elog(ERROR, "invalid string enlargement request size: %d", needed);
-   if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
+   if (((Size) needed) >= (limit - (Size) str->len))
        ereport(ERROR,
                (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                 errmsg("out of memory"),
@@ -261,7 +303,7 @@ enlargeStringInfo(StringInfo str, int needed)
 
    needed += str->len + 1;     /* total space required now */
 
-   /* Because of the above test, we now have needed <= MaxAllocSize */
+   /* Because of the above test, we now have needed <= limit */
 
    if (needed <= str->maxlen)
        return;                 /* got enough space already */
@@ -276,14 +318,14 @@ enlargeStringInfo(StringInfo str, int needed)
        newlen = 2 * newlen;
 
    /*
-    * Clamp to MaxAllocSize in case we went past it.  Note we are assuming
-    * here that MaxAllocSize <= INT_MAX/2, else the above loop could
-    * overflow.  We will still have newlen >= needed.
+    * Clamp to the limit in case we went past it.  Note we are assuming here
+    * that limit <= INT_MAX/2, else the above loop could overflow.  We will
+    * still have newlen >= needed.
     */
-   if (newlen > (int) MaxAllocSize)
-       newlen = (int) MaxAllocSize;
+   if (newlen > limit)
+       newlen = limit;
 
-   str->data = (char *) repalloc(str->data, newlen);
+   str->data = (char *) repalloc_huge(str->data, (Size) newlen);
 
    str->maxlen = newlen;
 }

diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h
index 00c25b6edd5a75aa2a35b78701fe2d74d955f8e8..07239dcedfc23acf80d501ac5c586185ed328865 100644 (file)
--- a/src/include/lib/stringinfo.h
+++ b/src/include/lib/stringinfo.h
@@ -30,6 +30,8 @@
  *     cursor  is initialized to zero by makeStringInfo or initStringInfo,
  *             but is not otherwise touched by the stringinfo.c routines.
  *             Some routines use it to scan through a StringInfo.
+ *     long_ok whether this StringInfo can allocate more than MaxAllocSize
+ *             bytes (but still up to 2GB).
  *-------------------------
  */
 typedef struct StringInfoData
@@ -38,6 +40,7 @@ typedef struct StringInfoData
    int         len;
    int         maxlen;
    int         cursor;
+   bool        long_ok;
 } StringInfoData;
 
 typedef StringInfoData *StringInfo;
@@ -46,11 +49,11 @@ typedef StringInfoData *StringInfo;
 /*------------------------
  * There are two ways to create a StringInfo object initially:
  *
- * StringInfo stringptr = makeStringInfo();
+ * StringInfo stringptr = makeStringInfo(); // or makeLongStringInfo();
  *     Both the StringInfoData and the data buffer are palloc'd.
  *
  * StringInfoData string;
- * initStringInfo(&string);
+ * initStringInfo(&string); // or initLongStringInfo();
  *     The data buffer is palloc'd but the StringInfoData is just local.
  *     This is the easiest approach for a StringInfo object that will
  *     only live as long as the current routine.
@@ -67,21 +70,26 @@ typedef StringInfoData *StringInfo;
 
 /*------------------------
  * makeStringInfo
- * Create an empty 'StringInfoData' & return a pointer to it.
+ * makeLongStringInfo
+ * Create an empty 'StringInfoData' & return a pointer to it.  The former
+ * allows up to 1 GB in size, per palloc(); the latter allows up to 2 GB.
  */
 extern StringInfo makeStringInfo(void);
+extern StringInfo makeLongStringInfo(void);
 
 /*------------------------
  * initStringInfo
+ * initLongStringInfo
  * Initialize a StringInfoData struct (with previously undefined contents)
- * to describe an empty string.
+ * to describe an empty string.  Size limits as above.
  */
 extern void initStringInfo(StringInfo str);
+extern void initLongStringInfo(StringInfo str);
 
 /*------------------------
  * resetStringInfo
  * Clears the current content of the StringInfo, if any. The
- * StringInfo remains valid.
+ * StringInfo remains valid.  The long_ok flag is not reset.
  */
 extern void resetStringInfo(StringInfo str);