Fix lexing of standard multi-character operators in edge cases.

Commits c6b3c939b (which fixed the precedence of >=, <=, <> operators)
and 865f14a2d (which added support for the standard => notation for
named arguments) created a class of lexer tokens which look like
multi-character operators but which have their own token IDs distinct
from Op. However, longest-match rules meant that following any of
these tokens with another operator character, as in (1<>-1), would
cause them to be incorrectly returned as Op.

The error here isn't immediately obvious, because the parser would
usually still find the correct operator via the Op token, but there
were more subtle problems:

1. If immediately followed by a comment or +-, >= <= <> would be given
   the old precedence of Op rather than the correct new precedence;

2. If followed by a comment, != would be returned as Op rather than as
   NOT_EQUAL, causing it not to be found at all;

3. If followed by a comment or +-, the => token for named arguments
   would be lexed as Op, causing the argument to be mis-parsed as a
   simple expression, usually causing an error.

Fix by explicitly checking for the operators in the {operator} code
block in addition to all the existing special cases there.

Backpatch to 9.5 where the problem was introduced.

Analysis and patch by me; review by Tom Lane.
Discussion: https://postgr.es/m/87va851ppl.fsf@news-spur.riddles.org.uk

diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index bcc6a91e044155eeb9155e43fdbdcfa217a6f4ea..c86a37b6a3cf47dc3dab5654c9d004af2f5e4948 100644 (file)
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -336,6 +336,15 @@ identifier     {ident_start}{ident_cont}*
 typecast       "::"
 dot_dot            \.\.
 colon_equals   ":="
+
+/*
+ * These operator-like tokens (unlike the above ones) also match the {operator}
+ * rule, which means that they might be overridden by a longer match if they
+ * are followed by a comment start or a + or - character. Accordingly, if you
+ * add to this list, you must also add corresponding code to the {operator}
+ * block to return the correct token in such cases. (This is not needed in
+ * psqlscan.l since the token value is ignored there.)
+ */
 equals_greater "=>"
 less_equals        "<="
 greater_equals ">="
@@ -921,6 +930,25 @@ other          .
                        if (nchars == 1 &&
                            strchr(",()[].;:+-*/%^<>=", yytext[0]))
                            return yytext[0];
+                       /*
+                        * Likewise, if what we have left is two chars, and
+                        * those match the tokens ">=", "<=", "=>", "<>" or
+                        * "!=", then we must return the appropriate token
+                        * rather than the generic Op.
+                        */
+                       if (nchars == 2)
+                       {
+                           if (yytext[0] == '=' && yytext[1] == '>')
+                               return EQUALS_GREATER;
+                           if (yytext[0] == '>' && yytext[1] == '=')
+                               return GREATER_EQUALS;
+                           if (yytext[0] == '<' && yytext[1] == '=')
+                               return LESS_EQUALS;
+                           if (yytext[0] == '<' && yytext[1] == '>')
+                               return NOT_EQUALS;
+                           if (yytext[0] == '!' && yytext[1] == '=')
+                               return NOT_EQUALS;
+                       }
                    }
 
                    /*

diff --git a/src/bin/psql/psqlscan.l b/src/bin/psql/psqlscan.l
index cde09c3e9c83f2112f1e1c71755a6b6ac314261a..a8f2bbe094a8e6e684901a21605ace6d76b54341 100644 (file)
--- a/src/bin/psql/psqlscan.l
+++ b/src/bin/psql/psqlscan.l
@@ -357,6 +357,15 @@ identifier     {ident_start}{ident_cont}*
 typecast       "::"
 dot_dot            \.\.
 colon_equals   ":="
+
+/*
+ * These operator-like tokens (unlike the above ones) also match the {operator}
+ * rule, which means that they might be overridden by a longer match if they
+ * are followed by a comment start or a + or - character. Accordingly, if you
+ * add to this list, you must also add corresponding code to the {operator}
+ * block to return the correct token in such cases. (This is not needed in
+ * psqlscan.l since the token value is ignored there.)
+ */
 equals_greater "=>"
 less_equals        "<="
 greater_equals ">="

diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l
index 1ce5ae5cf51217c059ce69a1177a33b84c5a450e..af0ba44de8572e528b5b96091cb9613ba120bd1f 100644 (file)
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -242,6 +242,15 @@ array          ({ident_cont}|{whitespace}|[\[\]\+\-\*\%\/\(\)\>\.])*
 typecast       "::"
 dot_dot            \.\.
 colon_equals   ":="
+
+/*
+ * These operator-like tokens (unlike the above ones) also match the {operator}
+ * rule, which means that they might be overridden by a longer match if they
+ * are followed by a comment start or a + or - character. Accordingly, if you
+ * add to this list, you must also add corresponding code to the {operator}
+ * block to return the correct token in such cases. (This is not needed in
+ * psqlscan.l since the token value is ignored there.)
+ */
 equals_greater "=>"
 less_equals        "<="
 greater_equals ">="
@@ -729,6 +738,25 @@ cppline            {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})(.*\\{space})*.
                            if (nchars == 1 &&
                                strchr(",()[].;:+-*/%^<>=", yytext[0]))
                                return yytext[0];
+                           /*
+                            * Likewise, if what we have left is two chars, and
+                            * those match the tokens ">=", "<=", "=>", "<>" or
+                            * "!=", then we must return the appropriate token
+                            * rather than the generic Op.
+                            */
+                           if (nchars == 2)
+                           {
+                               if (yytext[0] == '=' && yytext[1] == '>')
+                                   return EQUALS_GREATER;
+                               if (yytext[0] == '>' && yytext[1] == '=')
+                                   return GREATER_EQUALS;
+                               if (yytext[0] == '<' && yytext[1] == '=')
+                                   return LESS_EQUALS;
+                               if (yytext[0] == '<' && yytext[1] == '>')
+                                   return NOT_EQUALS;
+                               if (yytext[0] == '!' && yytext[1] == '=')
+                                   return NOT_EQUALS;
+                           }
                        }
 
                        base_yylval.str = mm_strdup(yytext);

diff --git a/src/test/regress/expected/create_operator.out b/src/test/regress/expected/create_operator.out
index 3a216c2ca8fd1eb58e3385ba4d019596eb29c70f..f3e41777318418cdc35a3365a98aa7ffa6283851 100644 (file)
--- a/src/test/regress/expected/create_operator.out
+++ b/src/test/regress/expected/create_operator.out
@@ -37,6 +37,80 @@ CREATE OPERATOR => (
 ERROR:  syntax error at or near "=>"
 LINE 1: CREATE OPERATOR => (
                         ^
+-- lexing of <=, >=, <>, != has a number of edge cases
+-- (=> is tested elsewhere)
+-- this is legal because ! is not allowed in sql ops
+CREATE OPERATOR !=- (
+   leftarg = int8,     -- right unary
+   procedure = numeric_fac
+);
+SELECT 2 !=-;
+ ?column? 
+----------
+        2
+(1 row)
+
+-- make sure lexer returns != as <> even in edge cases
+SELECT 2 !=/**/ 1, 2 !=/**/ 2;
+ ?column? | ?column? 
+----------+----------
+ t        | f
+(1 row)
+
+SELECT 2 !=-- comment to be removed by psql
+  1;
+ ?column? 
+----------
+ t
+(1 row)
+
+DO $$ -- use DO to protect -- from psql
+  declare r boolean;
+  begin
+    execute $e$ select 2 !=-- comment
+      1 $e$ into r;
+    raise info 'r = %', r;
+  end;
+$$;
+INFO:  r = t
+-- check that <= etc. followed by more operator characters are returned
+-- as the correct token with correct precedence
+SELECT true<>-1 BETWEEN 1 AND 1;  -- BETWEEN has prec. above <> but below Op
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT false<>/**/1 BETWEEN 1 AND 1;
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT false<=-1 BETWEEN 1 AND 1;
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT false>=-1 BETWEEN 1 AND 1;
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT 2<=/**/3, 3>=/**/2, 2<>/**/3;
+ ?column? | ?column? | ?column? 
+----------+----------+----------
+ t        | t        | t
+(1 row)
+
+SELECT 3<=/**/2, 2>=/**/3, 2<>/**/2;
+ ?column? | ?column? | ?column? 
+----------+----------+----------
+ f        | f        | f
+(1 row)
+
 -- Should fail. CREATE OPERATOR requires USAGE on SCHEMA
 BEGIN TRANSACTION;
 CREATE ROLE regress_rol_op1;

diff --git a/src/test/regress/expected/polymorphism.out b/src/test/regress/expected/polymorphism.out
index ddf45cf59734d8b5a1ee08d1c97bcd01a78106a3..24a13dc2680dbffa49dc078b0e5307df0dbc2f83 100644 (file)
--- a/src/test/regress/expected/polymorphism.out
+++ b/src/test/regress/expected/polymorphism.out
@@ -1423,6 +1423,42 @@ select dfunc('a'::text, 'b', flag => true); -- mixed notation
  a
 (1 row)
 
+-- this tests lexer edge cases around =>
+select dfunc(a =>-1);
+ dfunc 
+-------
+    -1
+(1 row)
+
+select dfunc(a =>+1);
+ dfunc 
+-------
+     1
+(1 row)
+
+select dfunc(a =>/**/1);
+ dfunc 
+-------
+     1
+(1 row)
+
+select dfunc(a =>--comment to be removed by psql
+  1);
+ dfunc 
+-------
+     1
+(1 row)
+
+-- need DO to protect the -- from psql
+do $$
+  declare r integer;
+  begin
+    select dfunc(a=>-- comment
+      1) into r;
+    raise info 'r = %', r;
+  end;
+$$;
+INFO:  r = 1
 -- check reverse-listing of named-arg calls
 CREATE VIEW dfview AS
    SELECT q1, q2,

diff --git a/src/test/regress/sql/create_operator.sql b/src/test/regress/sql/create_operator.sql
index 0e5d6356bc39e0c91e9907921f29fc9349e11b7c..0579cb95e1d425c55dd89b9967c199f435309793 100644 (file)
--- a/src/test/regress/sql/create_operator.sql
+++ b/src/test/regress/sql/create_operator.sql
@@ -41,6 +41,37 @@ CREATE OPERATOR => (
    procedure = numeric_fac
 );
 
+-- lexing of <=, >=, <>, != has a number of edge cases
+-- (=> is tested elsewhere)
+
+-- this is legal because ! is not allowed in sql ops
+CREATE OPERATOR !=- (
+   leftarg = int8,     -- right unary
+   procedure = numeric_fac
+);
+SELECT 2 !=-;
+-- make sure lexer returns != as <> even in edge cases
+SELECT 2 !=/**/ 1, 2 !=/**/ 2;
+SELECT 2 !=-- comment to be removed by psql
+  1;
+DO $$ -- use DO to protect -- from psql
+  declare r boolean;
+  begin
+    execute $e$ select 2 !=-- comment
+      1 $e$ into r;
+    raise info 'r = %', r;
+  end;
+$$;
+
+-- check that <= etc. followed by more operator characters are returned
+-- as the correct token with correct precedence
+SELECT true<>-1 BETWEEN 1 AND 1;  -- BETWEEN has prec. above <> but below Op
+SELECT false<>/**/1 BETWEEN 1 AND 1;
+SELECT false<=-1 BETWEEN 1 AND 1;
+SELECT false>=-1 BETWEEN 1 AND 1;
+SELECT 2<=/**/3, 3>=/**/2, 2<>/**/3;
+SELECT 3<=/**/2, 2>=/**/3, 2<>/**/2;
+
 -- Should fail. CREATE OPERATOR requires USAGE on SCHEMA
 BEGIN TRANSACTION;
 CREATE ROLE regress_rol_op1;

diff --git a/src/test/regress/sql/polymorphism.sql b/src/test/regress/sql/polymorphism.sql
index 72f6cb5e7cb16a4c162e6e38b9126fc17647bd9f..1b4c68ba9676af2dfde6189a42b166bf9f1aa13f 100644 (file)
--- a/src/test/regress/sql/polymorphism.sql
+++ b/src/test/regress/sql/polymorphism.sql
@@ -763,6 +763,21 @@ select dfunc('a'::text, 'b', flag => false); -- mixed notation
 select dfunc('a'::text, 'b', true); -- full positional notation
 select dfunc('a'::text, 'b', flag => true); -- mixed notation
 
+-- this tests lexer edge cases around =>
+select dfunc(a =>-1);
+select dfunc(a =>+1);
+select dfunc(a =>/**/1);
+select dfunc(a =>--comment to be removed by psql
+  1);
+-- need DO to protect the -- from psql
+do $$
+  declare r integer;
+  begin
+    select dfunc(a=>-- comment
+      1) into r;
+    raise info 'r = %', r;
+  end;
+$$;
 
 -- check reverse-listing of named-arg calls
 CREATE VIEW dfview AS