Make SQL arrays support null elements. This commit fixes the core array

functionality, but I still need to make another pass looking at places
that incidentally use arrays (such as ACL manipulation) to make sure they
are null-safe.  Contrib needs work too.
I have not changed the behaviors that are still under discussion about
array comparison and what to do with lower bounds.

Author: tglsfdc

contrib/intagg/int_aggregate.c

@@ -87,7 +87,7 @@ GetPGArray(PGARRAY * p, AggState *aggstate, bool fAdd)
 		p = (PGARRAY *) MemoryContextAlloc(aggstate->aggcontext, cb);
 		p->a.size = cb;
 		p->a.ndim = 1;
-		p->a.flags = 0;
+		p->a.dataoffset = 0;	/* we don't support nulls, for now */
 		p->a.elemtype = INT4OID;
 		p->items = 0;
 		p->lower = START_NUM;

contrib/intarray/_int_tool.c

@@ -208,12 +208,13 @@ ArrayType *
 new_intArrayType(int num)
 {
 	ArrayType  *r;
-	int			nbytes = ARR_OVERHEAD(NDIM) + sizeof(int) * num;
+	int			nbytes = ARR_OVERHEAD_NONULLS(NDIM) + sizeof(int) * num;
 
 	r = (ArrayType *) palloc0(nbytes);
 
 	ARR_SIZE(r) = nbytes;
 	ARR_NDIM(r) = NDIM;
+	r->dataoffset = 0;			/* marker for no null bitmap */
 	ARR_ELEMTYPE(r) = INT4OID;
 	*((int *) ARR_DIMS(r)) = num;
 	*((int *) ARR_LBOUND(r)) = 1;
@@ -224,7 +225,7 @@ new_intArrayType(int num)
 ArrayType *
 resize_intArrayType(ArrayType *a, int num)
 {
-	int			nbytes = ARR_OVERHEAD(NDIM) + sizeof(int) * num;
+	int			nbytes = ARR_OVERHEAD_NONULLS(NDIM) + sizeof(int) * num;
 
 	if (num == ARRNELEMS(a))
 		return a;

contrib/tsearch2/query_rewrite.c

@@ -232,7 +232,7 @@ rewrite_accum(PG_FUNCTION_ARGS) {
 	if (ARR_ELEMTYPE(qa) != tsqOid)
 		elog(ERROR, "array should contain tsquery type");
 
-	deconstruct_array(qa, tsqOid, -1, false, 'i', &elemsp, &nelemsp); 
+	deconstruct_array(qa, tsqOid, -1, false, 'i', &elemsp, NULL, &nelemsp); 
 
 	q = (QUERYTYPE*)DatumGetPointer( elemsp[0] );
 	if ( q->size == 0 ) {

doc/src/sgml/array.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/array.sgml,v 1.46 2005/11/04 23:13:59 petere Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/array.sgml,v 1.47 2005/11/17 22:14:50 tgl Exp $ -->
 
 <sect1 id="arrays">
  <title>Arrays</title>
@@ -110,6 +110,13 @@ CREATE TABLE tictactoe (
    three subarrays of integers.
   </para>
 
+  <para>
+   To set an element of an array constant to NULL, write <literal>NULL</>
+   for the element value.  (Any upper- or lower-case variant of
+   <literal>NULL</> will do.)  If you want an actual string value
+   <quote>NULL</>, you must put double quotes around it.
+  </para>
+
   <para>
    (These kinds of array constants are actually only a special case of
    the generic type constants discussed in <xref
@@ -121,17 +128,6 @@ CREATE TABLE tictactoe (
   <para>
    Now we can show some <command>INSERT</command> statements.
 
-<programlisting>
-INSERT INTO sal_emp
-    VALUES ('Bill',
-    '{10000, 10000, 10000, 10000}',
-    '{{"meeting", "lunch"}, {"meeting"}}');
-ERROR:  multidimensional arrays must have array expressions with matching dimensions
-</programlisting>
-
-  Note that multidimensional arrays must have matching extents for each
-  dimension. A mismatch causes an error report.
-
 <programlisting>
 INSERT INTO sal_emp
     VALUES ('Bill',
@@ -145,15 +141,9 @@ INSERT INTO sal_emp
 </programlisting>
   </para>
 
-  <para>
-   A limitation of the present array implementation is that individual
-   elements of an array cannot be SQL null values.  The entire array
-   can be set to null, but you can't have an array with some elements
-   null and some not.  (This is likely to change in the future.)
-  </para>
-
  <para>
   The result of the previous two inserts looks like this:
+
 <programlisting>
 SELECT * FROM sal_emp;
  name  |      pay_by_quarter       |                 schedule
@@ -183,6 +173,19 @@ INSERT INTO sal_emp
   constructor syntax is discussed in more detail in
   <xref linkend="sql-syntax-array-constructors">.
  </para>
+
+ <para>
+  Multidimensional arrays must have matching extents for each
+  dimension. A mismatch causes an error report, for example:
+
+<programlisting>
+INSERT INTO sal_emp
+    VALUES ('Bill',
+    '{10000, 10000, 10000, 10000}',
+    '{{"meeting", "lunch"}, {"meeting"}}');
+ERROR:  multidimensional arrays must have array expressions with matching dimensions
+</programlisting>
+ </para>
  </sect2>
 
  <sect2>
@@ -262,14 +265,22 @@ SELECT schedule[1:2][2] FROM sal_emp WHERE name = 'Bill';
  </para>
 
  <para>
-  Fetching from outside the current bounds of an array yields a
-  SQL null value, not an error.  For example, if <literal>schedule</>
+  An array subscript expression will return null if either the array itself or
+  any of the subscript expressions are null.  Also, null is returned if a
+  subscript is outside the array bounds (this case does not raise an error).
+  For example, if <literal>schedule</>
   currently has the dimensions <literal>[1:3][1:2]</> then referencing
   <literal>schedule[3][3]</> yields NULL.  Similarly, an array reference
   with the wrong number of subscripts yields a null rather than an error.
-  Fetching an array slice that
-  is completely outside the current bounds likewise yields a null array;
-  but if the requested slice partially overlaps the array bounds, then it
+ </para>
+
+ <para>
+  An array slice expression likewise yields null if the array itself or
+  any of the subscript expressions are null.  However, in other corner
+  cases such as selecting an array slice that
+  is completely outside the current array bounds, a slice expression
+  yields an empty (zero-dimensional) array instead of null.
+  If the requested slice partially overlaps the array bounds, then it
   is silently reduced to just the overlapping region.
  </para>
 
@@ -349,7 +360,7 @@ UPDATE sal_emp SET pay_by_quarter[1:2] = '{27000,27000}'
  </para>
 
  <para>
-  Array slice assignment allows creation of arrays that do not use one-based
+  Subscripted assignment allows creation of arrays that do not use one-based
   subscripts.  For example one might assign to <literal>myarray[-2:7]</> to
   create an array with subscript values running from -2 to 7.
  </para>
@@ -442,7 +453,7 @@ SELECT array_dims(ARRAY[1,2] || ARRAY[[3,4],[5,6]]);
   arrays, but <function>array_cat</function> supports multidimensional arrays.
 
   Note that the concatenation operator discussed above is preferred over
-  direct use of these functions. In fact, the functions are primarily for use
+  direct use of these functions. In fact, the functions exist primarily for use
   in implementing the concatenation operator. However, they may be directly
   useful in the creation of user-defined aggregates. Some examples:
 
@@ -544,8 +555,9 @@ SELECT * FROM sal_emp WHERE 10000 = ALL (pay_by_quarter);
 
   <para>
    The array output routine will put double quotes around element values
-   if they are empty strings or contain curly braces, delimiter characters,
-   double quotes, backslashes, or white space.  Double quotes and backslashes
+   if they are empty strings, contain curly braces, delimiter characters,
+   double quotes, backslashes, or white space, or match the word
+   <literal>NULL</>.  Double quotes and backslashes
    embedded in element values will be backslash-escaped.  For numeric
    data types it is safe to assume that double quotes will never appear, but
    for textual data types one should be prepared to cope with either presence
@@ -555,35 +567,15 @@ SELECT * FROM sal_emp WHERE 10000 = ALL (pay_by_quarter);
 
   <para>
    By default, the lower bound index value of an array's dimensions is
-   set to one. If any of an array's dimensions has a lower bound index not
-   equal to one, an additional decoration that indicates the actual
-   array dimensions will precede the array structure decoration.
+   set to one.  To represent arrays with other lower bounds, the array
+   subscript ranges can be specified explicitly before writing the
+   array contents.
    This decoration consists of square brackets (<literal>[]</>)
    around each array dimension's lower and upper bounds, with
    a colon (<literal>:</>) delimiter character in between. The
    array dimension decoration is followed by an equal sign (<literal>=</>).
    For example:
 <programlisting>
-SELECT 1 || ARRAY[2,3] AS array;
-
-     array
----------------
- [0:2]={1,2,3}
-(1 row)
-
-SELECT ARRAY[1,2] || ARRAY[[3,4]] AS array;
-
-          array
---------------------------
- [0:1][1:2]={{1,2},{3,4}}
-(1 row)
-</programlisting>
-  </para>
-
-  <para>
-   This syntax can also be used to specify non-default array subscripts
-   in an array literal. For example:
-<programlisting>
 SELECT f1[1][-2][3] AS e1, f1[1][-1][5] AS e2
  FROM (SELECT '[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int[] AS f1) AS ss;
 
@@ -592,6 +584,18 @@ SELECT f1[1][-2][3] AS e1, f1[1][-1][5] AS e2
   1 |  6
 (1 row)
 </programlisting>
+   The array output routine will include explicit dimensions in its result
+   only when there are one or more lower bounds different from one.
+  </para>
+
+  <para>
+   If the value written for an element is <literal>NULL</> (in any case
+   variant), the element is taken to be NULL.  The presence of any quotes
+   or backslashes disables this and allows the literal string value
+   <quote>NULL</> to be entered.  Also, for backwards compatibility with
+   pre-8.2 versions of <productname>PostgreSQL</>, the <xref
+   linkend="guc-array-nulls"> configuration parameter may be turned
+   <literal>off</> to suppress recognition of <literal>NULL</> as a NULL.
   </para>
 
   <para>
@@ -600,7 +604,9 @@ SELECT f1[1][-2][3] AS e1, f1[1][-1][5] AS e2
    if the element value would otherwise confuse the array-value parser.
    For example, elements containing curly braces, commas (or whatever the
    delimiter character is), double quotes, backslashes, or leading or trailing
-   whitespace must be double-quoted.  To put a double quote or backslash in a
+   whitespace must be double-quoted.  Empty strings and strings matching the
+   word <literal>NULL</> must be quoted, too.  To put a double quote or
+   backslash in a
    quoted array element value, precede it with a backslash. Alternatively, you
    can use backslash-escaping to protect all data characters that would
    otherwise be taken as array syntax.

doc/src/sgml/config.sgml

@@ -1,5 +1,5 @@
 <!--
-$PostgreSQL: pgsql/doc/src/sgml/config.sgml,v 1.36 2005/11/04 23:53:18 tgl Exp $
+$PostgreSQL: pgsql/doc/src/sgml/config.sgml,v 1.37 2005/11/17 22:14:50 tgl Exp $
 -->
 <chapter Id="runtime-config">
   <title>Server Configuration</title>
@@ -3614,6 +3614,7 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
 
     <sect2 id="runtime-config-compatible-version">
      <title>Previous PostgreSQL Versions</title>
+
      <variablelist>
 
      <varlistentry id="guc-add-missing-from" xreflabel="add_missing_from">
@@ -3647,40 +3648,27 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
       </listitem>
      </varlistentry>
 
-     <varlistentry id="guc-regex-flavor" xreflabel="regex_flavor">
-      <term><varname>regex_flavor</varname> (<type>string</type>)</term>
-      <indexterm><primary>regular expressions</></>
+     <varlistentry id="guc-array-nulls" xreflabel="array_nulls">
+      <term><varname>array_nulls</varname> (<type>boolean</type>)</term>
       <indexterm>
-       <primary><varname>regex_flavor</> configuration parameter</primary>
+       <primary><varname>array_nulls</> configuration parameter</primary>
       </indexterm>
       <listitem>
        <para>
-        The regular expression <quote>flavor</> can be set to
-        <literal>advanced</>, <literal>extended</>, or <literal>basic</>.
-        The default is <literal>advanced</>.  The <literal>extended</>
-        setting may be useful for exact backwards compatibility with
-        pre-7.4 releases of <productname>PostgreSQL</>.  See
-        <xref linkend="posix-syntax-details"> for details.
+        This controls whether the array input parser recognizes
+        unquoted <literal>NULL</> as specifying a NULL array element.
+        By default, this is <literal>on</>, allowing array values containing
+        NULLs to be entered.  However, <productname>PostgreSQL</> versions
+        before 8.2 did not support NULLs in arrays, and therefore would
+        treat <literal>NULL</> as specifying a normal array element with
+        the string value <quote>NULL</>.  For backwards compatibility with
+        applications that require the old behavior, this variable can be
+        turned <literal>off</>.
        </para>
-      </listitem>
-     </varlistentry>
 
-     <varlistentry id="guc-sql-inheritance" xreflabel="sql_inheritance">
-      <term><varname>sql_inheritance</varname> (<type>boolean</type>)</term>
-      <indexterm>
-       <primary><varname>sql_inheritance</> configuration parameter</primary>
-      </indexterm>
-      <indexterm><primary>inheritance</></>
-      <listitem>
        <para>
-        This controls the inheritance semantics, in particular whether
-        subtables are included by various commands by default. They were
-        not included in versions prior to 7.1. If you need the old
-        behavior you can set this variable to <literal>off</>, but in
-        the long run you are encouraged to change your applications to
-        use the <literal>ONLY</literal> key word to exclude subtables.
-        See <xref linkend="ddl-inherit"> for more information about
-        inheritance.
+        Note that it is possible to create array values containing NULLs
+        even when this variable is <literal>off</>.
        </para>
       </listitem>
      </varlistentry>
@@ -3736,8 +3724,47 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-regex-flavor" xreflabel="regex_flavor">
+      <term><varname>regex_flavor</varname> (<type>string</type>)</term>
+      <indexterm><primary>regular expressions</></>
+      <indexterm>
+       <primary><varname>regex_flavor</> configuration parameter</primary>
+      </indexterm>
+      <listitem>
+       <para>
+        The regular expression <quote>flavor</> can be set to
+        <literal>advanced</>, <literal>extended</>, or <literal>basic</>.
+        The default is <literal>advanced</>.  The <literal>extended</>
+        setting may be useful for exact backwards compatibility with
+        pre-7.4 releases of <productname>PostgreSQL</>.  See
+        <xref linkend="posix-syntax-details"> for details.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="guc-sql-inheritance" xreflabel="sql_inheritance">
+      <term><varname>sql_inheritance</varname> (<type>boolean</type>)</term>
+      <indexterm>
+       <primary><varname>sql_inheritance</> configuration parameter</primary>
+      </indexterm>
+      <indexterm><primary>inheritance</></>
+      <listitem>
+       <para>
+        This controls the inheritance semantics, in particular whether
+        subtables are included by various commands by default. They were
+        not included in versions prior to 7.1. If you need the old
+        behavior you can set this variable to <literal>off</>, but in
+        the long run you are encouraged to change your applications to
+        use the <literal>ONLY</literal> key word to exclude subtables.
+        See <xref linkend="ddl-inherit"> for more information about
+        inheritance.
+       </para>
+      </listitem>
+     </varlistentry>
+
      </variablelist>
     </sect2>
+
     <sect2 id="runtime-config-compatible-clients">
      <title>Platform and Client Compatibility</title>
      <variablelist>