Make 'col IS NULL' clauses be indexable conditions.

Teodor Sigaev, with some kibitzing from Tom Lane.

Author: tglsfdc

doc/src/sgml/catalogs.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/catalogs.sgml,v 2.149 2007/04/02 03:49:36 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/catalogs.sgml,v 2.150 2007/04/06 22:33:41 tgl Exp $ -->
 <!--
  Documentation of the system catalogs, directed toward PostgreSQL developers
  -->
@@ -405,6 +405,13 @@
       <entry>Does the access method support null index entries?</entry>
      </row>
 
+     <row>
+      <entry><structfield>amsearchnulls</structfield></entry>
+      <entry><type>bool</type></entry>
+      <entry></entry>
+      <entry>Does the access method support IS NULL searches?</entry>
+     </row>
+
      <row>
       <entry><structfield>amstorage</structfield></entry>
       <entry><type>bool</type></entry>

doc/src/sgml/indexam.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/indexam.sgml,v 2.22 2007/02/22 22:00:22 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/indexam.sgml,v 2.23 2007/04/06 22:33:41 tgl Exp $ -->
 
 <chapter id="indexam">
  <title>Index Access Method Interface Definition</title>
@@ -129,7 +129,10 @@
    It is, however, OK to omit rows where the first indexed column is null.
    Thus, <structfield>amindexnulls</structfield> should be set true only if the
    index access method indexes all rows, including arbitrary combinations of
-   null values.
+   null values.  An index access method that sets
+   <structfield>amindexnulls</structfield> may also set
+   <structfield>amsearchnulls</structfield>, indicating that it supports
+   <literal>IS NULL</> clauses as search conditions.
   </para>
 
  </sect1>

doc/src/sgml/indices.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/indices.sgml,v 1.70 2007/02/14 20:47:15 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/indices.sgml,v 1.71 2007/04/06 22:33:41 tgl Exp $ -->
 
 <chapter id="indexes">
  <title id="indexes-title">Indexes</title>
@@ -147,8 +147,8 @@ CREATE INDEX test1_id_index ON test1 (id);
 
    Constructs equivalent to combinations of these operators, such as
    <literal>BETWEEN</> and <literal>IN</>, can also be implemented with
-   a B-tree index search.  (But note that <literal>IS NULL</> is not
-   equivalent to <literal>=</> and is not indexable.)
+   a B-tree index search.  Also, an <literal>IS NULL</> condition on
+   an index column can be used with a B-tree index.
   </para>
 
   <para>
@@ -180,8 +180,9 @@ CREATE INDEX test1_id_index ON test1 (id);
    Hash indexes can only handle simple equality comparisons.
    The query planner will consider using a hash index whenever an
    indexed column is involved in a comparison using the
-   <literal>=</literal> operator.  The following command is used to
-   create a hash index:
+   <literal>=</literal> operator.  (But hash indexes do not support
+   <literal>IS NULL</> searches.)
+   The following command is used to create a hash index:
 <synopsis>
 CREATE INDEX <replaceable>name</replaceable> ON <replaceable>table</replaceable> USING hash (<replaceable>column</replaceable>);
 </synopsis>
@@ -234,6 +235,8 @@ CREATE INDEX <replaceable>name</replaceable> ON <replaceable>table</replaceable>
 
    (See <xref linkend="functions-geometry"> for the meaning of
    these operators.)
+   Also, an <literal>IS NULL</> condition on
+   an index column can be used with a GiST index.
    Many other GiST operator
    classes are available in the <literal>contrib</> collection or as separate
    projects.  For more information see <xref linkend="GiST">.
@@ -266,6 +269,7 @@ CREATE INDEX <replaceable>name</replaceable> ON <replaceable>table</replaceable>
 
    (See <xref linkend="functions-array"> for the meaning of
    these operators.)
+   GIN indexes cannot use <literal>IS NULL</> as a search condition.
    Other GIN operator classes are available in the <literal>contrib</>
    <literal>tsearch2</literal> and <literal>intarray</literal> modules.
    For more information see <xref linkend="GIN">.

doc/src/sgml/ref/create_index.sgml

@@ -1,5 +1,5 @@
 <!--
-$PostgreSQL: pgsql/doc/src/sgml/ref/create_index.sgml,v 1.61 2007/04/03 22:38:35 momjian Exp $
+$PostgreSQL: pgsql/doc/src/sgml/ref/create_index.sgml,v 1.62 2007/04/06 22:33:41 tgl Exp $
 PostgreSQL documentation
 -->
 
@@ -434,12 +434,6 @@ Indexes:
    to remove an index.
   </para>
 
-  <para>
-   Indexes are not used for <literal>IS NULL</> clauses by default.
-   The best way to use indexes in such cases is to create a partial index
-   using an <literal>IS NULL</> predicate.
-  </para>
-
   <para>
    Prior releases of <productname>PostgreSQL</productname> also had an
    R-tree index method.  This method has been removed because

src/backend/access/common/scankey.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/common/scankey.c,v 1.29 2007/01/05 22:19:21 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/common/scankey.c,v 1.30 2007/04/06 22:33:41 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -20,7 +20,8 @@
 /*
  * ScanKeyEntryInitialize
  *		Initializes a scan key entry given all the field values.
- *		The target procedure is specified by OID.
+ *		The target procedure is specified by OID (but can be invalid
+ *		if SK_SEARCHNULL is set).
  *
  * Note: CurrentMemoryContext at call should be as long-lived as the ScanKey
  * itself, because that's what will be used for any subsidiary info attached
@@ -40,7 +41,13 @@ ScanKeyEntryInitialize(ScanKey entry,
 	entry->sk_strategy = strategy;
 	entry->sk_subtype = subtype;
 	entry->sk_argument = argument;
-	fmgr_info(procedure, &entry->sk_func);
+	if (RegProcedureIsValid(procedure))
+		fmgr_info(procedure, &entry->sk_func);
+	else
+	{
+		Assert(flags & SK_SEARCHNULL);
+		MemSet(&entry->sk_func, 0, sizeof(entry->sk_func));
+	}
 }
 
 /*

src/backend/access/gist/gistget.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/gist/gistget.c,v 1.64 2007/01/20 18:43:35 neilc Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/gist/gistget.c,v 1.65 2007/04/06 22:33:41 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -381,37 +381,45 @@ gistindex_keytest(IndexTuple tuple,
 		if (key->sk_flags & SK_ISNULL)
 		{
 			/*
-			 * is the compared-to datum NULL? on non-leaf page it's possible
-			 * to have nulls in childs :(
+			 * On non-leaf page we can't conclude that child hasn't NULL
+			 * values because of assumption in GiST: uinon (VAL, NULL) is VAL
+			 * But if on non-leaf page key IS  NULL then all childs
+			 * has NULL.
 			 */
 
-			if (isNull || !GistPageIsLeaf(p))
-				return true;
-			return false;
+			Assert( key->sk_flags & SK_SEARCHNULL );
+
+			if ( GistPageIsLeaf(p) && !isNull )
+				return false;
 		}
 		else if (isNull)
+		{
 			return false;
+		}
+		else
+		{
 
-		gistdentryinit(giststate, key->sk_attno - 1, &de,
-					   datum, r, p, offset,
-					   FALSE, isNull);
+			gistdentryinit(giststate, key->sk_attno - 1, &de,
+						   datum, r, p, offset,
+						   FALSE, isNull);
 
-		/*
-		 * Call the Consistent function to evaluate the test.  The arguments
-		 * are the index datum (as a GISTENTRY*), the comparison datum, and
-		 * the comparison operator's strategy number and subtype from pg_amop.
-		 *
-		 * (Presently there's no need to pass the subtype since it'll always
-		 * be zero, but might as well pass it for possible future use.)
-		 */
-		test = FunctionCall4(&key->sk_func,
-							 PointerGetDatum(&de),
-							 key->sk_argument,
-							 Int32GetDatum(key->sk_strategy),
-							 ObjectIdGetDatum(key->sk_subtype));
-
-		if (!DatumGetBool(test))
-			return false;
+			/*
+			 * Call the Consistent function to evaluate the test.  The arguments
+			 * are the index datum (as a GISTENTRY*), the comparison datum, and
+			 * the comparison operator's strategy number and subtype from pg_amop.
+			 *
+			 * (Presently there's no need to pass the subtype since it'll always
+			 * be zero, but might as well pass it for possible future use.)
+			 */
+			test = FunctionCall4(&key->sk_func,
+								 PointerGetDatum(&de),
+								 key->sk_argument,
+								 Int32GetDatum(key->sk_strategy),
+								 ObjectIdGetDatum(key->sk_subtype));
+
+			if (!DatumGetBool(test))
+				return false;
+		}
 
 		keySize--;
 		key++;

src/backend/access/nbtree/nbtsearch.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtsearch.c,v 1.111 2007/01/09 02:14:10 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtsearch.c,v 1.112 2007/04/06 22:33:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -746,8 +746,6 @@ _bt_first(IndexScanDesc scan, ScanDirection dir)
 	 *
 	 * If goback = true, we will then step back one item, while if
 	 * goback = false, we will start the scan on the located item.
-	 *
-	 * it's yet other place to add some code later for is(not)null ...
 	 *----------
 	 */
 	switch (strat_total)

src/backend/access/nbtree/nbtutils.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtutils.c,v 1.83 2007/03/30 00:12:59 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtutils.c,v 1.84 2007/04/06 22:33:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -264,12 +264,27 @@ _bt_preprocess_keys(IndexScanDesc scan)
 	if (numberOfKeys == 1)
 	{
 		/*
-		 * We don't use indices for 'A is null' and 'A is not null' currently
-		 * and 'A < = > <> NULL' will always fail - so qual is not OK if
-		 * comparison value is NULL.	  - vadim 03/21/97
+		 * We treat all btree operators as strict (even if they're not so
+		 * marked in pg_proc).  This means that it is impossible for an
+		 * operator condition with a NULL comparison constant to succeed,
+		 * and we can reject it right away.
+		 *
+		 * However, we now also support "x IS NULL" clauses as search
+		 * conditions, so in that case keep going.  The planner has not
+		 * filled in any particular strategy in this case, so set it to
+		 * BTEqualStrategyNumber --- we can treat IS NULL as an equality
+		 * operator for purposes of search strategy.
 		 */
 		if (cur->sk_flags & SK_ISNULL)
-			so->qual_ok = false;
+		{
+			if (cur->sk_flags & SK_SEARCHNULL)
+			{
+				cur->sk_strategy = BTEqualStrategyNumber;
+				cur->sk_subtype = InvalidOid;
+			}
+			else
+				so->qual_ok = false;
+		}
 		_bt_mark_scankey_with_indoption(cur, indoption);
 		memcpy(outkeys, cur, sizeof(ScanKeyData));
 		so->numberOfKeys = 1;
@@ -303,17 +318,20 @@ _bt_preprocess_keys(IndexScanDesc scan)
 	{
 		if (i < numberOfKeys)
 		{
-			/* See comments above: any NULL implies cannot match qual */
+			/* See comments above about NULLs and IS NULL handling. */
 			/* Note: we assume SK_ISNULL is never set in a row header key */
 			if (cur->sk_flags & SK_ISNULL)
 			{
-				so->qual_ok = false;
-
-				/*
-				 * Quit processing so we don't try to invoke comparison
-				 * routines on NULLs.
-				 */
-				return;
+				if (cur->sk_flags & SK_SEARCHNULL)
+				{
+					cur->sk_strategy = BTEqualStrategyNumber;
+					cur->sk_subtype = InvalidOid;
+				}
+				else
+				{
+					so->qual_ok = false;
+					return;
+				}
 			}
 		}
 
@@ -344,6 +362,14 @@ _bt_preprocess_keys(IndexScanDesc scan)
 
 					if (!chk || j == (BTEqualStrategyNumber - 1))
 						continue;
+
+					/* IS NULL together with any other predicate must fail */
+					if (eq->sk_flags & SK_SEARCHNULL)
+					{
+						so->qual_ok = false;
+						return;
+					}
+
 					if (_bt_compare_scankey_args(scan, chk, eq, chk,
 												 &test_result))
 					{
@@ -455,6 +481,23 @@ _bt_preprocess_keys(IndexScanDesc scan)
 		else
 		{
 			/* yup, keep only the more restrictive key */
+
+			/* if either arg is NULL, don't try to compare */
+			if ((cur->sk_flags | xform[j]->sk_flags) & SK_ISNULL)
+			{
+				/* at least one of them must be an IS NULL clause */
+				Assert(j == (BTEqualStrategyNumber - 1));
+				Assert((cur->sk_flags | xform[j]->sk_flags) & SK_SEARCHNULL);
+				/* if one is and one isn't, the search must fail */
+				if ((cur->sk_flags ^ xform[j]->sk_flags) & SK_SEARCHNULL)
+				{
+					so->qual_ok = false;
+					return;
+				}
+				/* we have duplicate IS NULL clauses, ignore the newer one */
+				continue;
+			}
+
 			if (_bt_compare_scankey_args(scan, cur, cur, xform[j],
 										 &test_result))
 			{
@@ -798,11 +841,29 @@ _bt_checkkeys(IndexScanDesc scan,
 							  tupdesc,
 							  &isNull);
 
-		/* btree doesn't support 'A is null' clauses, yet */
 		if (key->sk_flags & SK_ISNULL)
 		{
-			/* we shouldn't get here, really; see _bt_preprocess_keys() */
-			*continuescan = false;
+			/* Handle IS NULL tests */
+			Assert(key->sk_flags & SK_SEARCHNULL);
+
+			if (isNull)
+				continue;		/* tuple satisfies this qual */
+
+			/*
+			 * Tuple fails this qual.  If it's a required qual for the current
+			 * scan direction, then we can conclude no further tuples will
+			 * pass, either.
+			 */
+			if ((key->sk_flags & SK_BT_REQFWD) &&
+				ScanDirectionIsForward(dir))
+				*continuescan = false;
+			else if ((key->sk_flags & SK_BT_REQBKWD) &&
+					 ScanDirectionIsBackward(dir))
+				*continuescan = false;
+
+			/*
+			 * In any case, this indextuple doesn't match the qual.
+			 */
 			return false;
 		}