Prevent mis-encoding of "trailing junk after numeric literal" errors.

Since commit 2549f0661, we reject an identifier immediately following
a numeric literal (without separating whitespace), because that risks
ambiguity with hex/octal/binary integers.  However, that patch used
token patterns like "{integer}{ident_start}", which is problematic
because {ident_start} matches only a single byte.  If the first
character after the integer is a multibyte character, this ends up
with flex reporting an error message that includes a partial multibyte
character.  That can cause assorted bad-encoding problems downstream,
both in the report to the client and in the postmaster log file.

To fix, use {identifier} not {ident_start} in the "junk" token
patterns, so that they will match complete multibyte characters.
This seems generally better user experience quite aside from the
encoding problem: for "123abc" the error message will now say that
the error appeared at or near "123abc" instead of "123a".

While at it, add some commentary about why these patterns exist
and how they work.

Report and patch by Karina Litskevich; review by Pavel Borisov.
Back-patch to v15 where the problem came in.

Discussion: https://postgr.es/m/CACiT8iZ_diop=0zJ7zuY3BXegJpkKK1Av-PU7xh0EDYHsa5+=g@mail.gmail.com

diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index 54c54f4313b5d420187c4950976833cc47c3a815..8a678f068e66fa90663a1bc6a0a66c1f9fc64304 100644 (file)
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -412,16 +412,30 @@ numericfail       {decinteger}\.\.
 real           ({decinteger}|{numeric})[Ee][-+]?{decinteger}
 realfail       ({decinteger}|{numeric})[Ee][-+]
 
-decinteger_junk    {decinteger}{ident_start}
-hexinteger_junk    {hexinteger}{ident_start}
-octinteger_junk    {octinteger}{ident_start}
-bininteger_junk    {bininteger}{ident_start}
-numeric_junk   {numeric}{ident_start}
-real_junk      {real}{ident_start}
-
 /* Positional parameters don't accept underscores. */
 param          \${decdigit}+
-param_junk     \${decdigit}+{ident_start}
+
+/*
+ * An identifier immediately following an integer literal is disallowed because
+ * in some cases it's ambiguous what is meant: for example, 0x1234 could be
+ * either a hexinteger or a decinteger "0" and an identifier "x1234".  We can
+ * detect such problems by seeing if integer_junk matches a longer substring
+ * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
+ * bininteger).  One "junk" pattern is sufficient because
+ * {decinteger}{identifier} will match all the same strings we'd match with
+ * {hexinteger}{identifier} etc.
+ *
+ * Note that the rule for integer_junk must appear after the ones for
+ * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
+ * and integer_junk, and we need hexinteger to be chosen in that case.
+ *
+ * Also disallow strings matched by numeric_junk, real_junk and param_junk
+ * for consistency.
+ */
+integer_junk   {decinteger}{identifier}
+numeric_junk   {numeric}{identifier}
+real_junk      {real}{identifier}
+param_junk     \${decdigit}+{identifier}
 
 other          .
 
@@ -1049,19 +1063,7 @@ other            .
                    SET_YYLLOC();
                    yyerror("trailing junk after numeric literal");
                }
-{decinteger_junk}  {
-                   SET_YYLLOC();
-                   yyerror("trailing junk after numeric literal");
-               }
-{hexinteger_junk}  {
-                   SET_YYLLOC();
-                   yyerror("trailing junk after numeric literal");
-               }
-{octinteger_junk}  {
-                   SET_YYLLOC();
-                   yyerror("trailing junk after numeric literal");
-               }
-{bininteger_junk}  {
+{integer_junk} {
                    SET_YYLLOC();
                    yyerror("trailing junk after numeric literal");
                }

diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index 1be5a9f10b8c03f2629c222a505d1a209e946d65..0f20903ef5b80969157356bc64d22a6b817da591 100644 (file)
--- a/src/fe_utils/psqlscan.l
+++ b/src/fe_utils/psqlscan.l
@@ -348,16 +348,30 @@ numericfail       {decinteger}\.\.
 real           ({decinteger}|{numeric})[Ee][-+]?{decinteger}
 realfail       ({decinteger}|{numeric})[Ee][-+]
 
-decinteger_junk    {decinteger}{ident_start}
-hexinteger_junk    {hexinteger}{ident_start}
-octinteger_junk    {octinteger}{ident_start}
-bininteger_junk    {bininteger}{ident_start}
-numeric_junk   {numeric}{ident_start}
-real_junk      {real}{ident_start}
-
 /* Positional parameters don't accept underscores. */
 param          \${decdigit}+
-param_junk     \${decdigit}+{ident_start}
+
+/*
+ * An identifier immediately following an integer literal is disallowed because
+ * in some cases it's ambiguous what is meant: for example, 0x1234 could be
+ * either a hexinteger or a decinteger "0" and an identifier "x1234".  We can
+ * detect such problems by seeing if integer_junk matches a longer substring
+ * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
+ * bininteger).  One "junk" pattern is sufficient because
+ * {decinteger}{identifier} will match all the same strings we'd match with
+ * {hexinteger}{identifier} etc.
+ *
+ * Note that the rule for integer_junk must appear after the ones for
+ * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
+ * and integer_junk, and we need hexinteger to be chosen in that case.
+ *
+ * Also disallow strings matched by numeric_junk, real_junk and param_junk
+ * for consistency.
+ */
+integer_junk   {decinteger}{identifier}
+numeric_junk   {numeric}{identifier}
+real_junk      {real}{identifier}
+param_junk     \${decdigit}+{identifier}
 
 /* psql-specific: characters allowed in variable names */
 variable_char  [A-Za-z\200-\377_0-9]
@@ -898,16 +912,7 @@ other          .
 {realfail}     {
                    ECHO;
                }
-{decinteger_junk}  {
-                   ECHO;
-               }
-{hexinteger_junk}  {
-                   ECHO;
-               }
-{octinteger_junk}  {
-                   ECHO;
-               }
-{bininteger_junk}  {
+{integer_junk} {
                    ECHO;
                }
 {numeric_junk} {

diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l
index ba523e69129ed67b086a700876880a212d2bd8eb..49c0e7d68e72c616e58759be6d628974f41f0a8b 100644 (file)
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -381,16 +381,30 @@ numericfail       {decinteger}\.\.
 real           ({decinteger}|{numeric})[Ee][-+]?{decinteger}
 realfail       ({decinteger}|{numeric})[Ee][-+]
 
-decinteger_junk    {decinteger}{ident_start}
-hexinteger_junk    {hexinteger}{ident_start}
-octinteger_junk    {octinteger}{ident_start}
-bininteger_junk    {bininteger}{ident_start}
-numeric_junk   {numeric}{ident_start}
-real_junk      {real}{ident_start}
-
 /* Positional parameters don't accept underscores. */
 param          \${decdigit}+
-param_junk     \${decdigit}+{ident_start}
+
+/*
+ * An identifier immediately following an integer literal is disallowed because
+ * in some cases it's ambiguous what is meant: for example, 0x1234 could be
+ * either a hexinteger or a decinteger "0" and an identifier "x1234".  We can
+ * detect such problems by seeing if integer_junk matches a longer substring
+ * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
+ * bininteger).  One "junk" pattern is sufficient because
+ * {decinteger}{identifier} will match all the same strings we'd match with
+ * {hexinteger}{identifier} etc.
+ *
+ * Note that the rule for integer_junk must appear after the ones for
+ * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
+ * and integer_junk, and we need hexinteger to be chosen in that case.
+ *
+ * Also disallow strings matched by numeric_junk, real_junk and param_junk
+ * for consistency.
+ */
+integer_junk   {decinteger}{identifier}
+numeric_junk   {numeric}{identifier}
+real_junk      {real}{identifier}
+param_junk     \${decdigit}+{identifier}
 
 /* special characters for other dbms */
 /* we have to react differently in compat mode */
@@ -993,16 +1007,7 @@ cppline           {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
     * Note that some trailing junk is valid in C (such as 100LL), so we
     * contain this to SQL mode.
     */
-{decinteger_junk}  {
-                   mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
-               }
-{hexinteger_junk}  {
-                   mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
-               }
-{octinteger_junk}  {
-                   mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
-               }
-{bininteger_junk}  {
+{integer_junk} {
                    mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
                }
 {numeric_junk} {

diff --git a/src/test/regress/expected/numerology.out b/src/test/regress/expected/numerology.out
index 8d4a3ba228a1c3b79632cd55ce6e57bd689d0b8f..3512a1d04ce5ef55387bb75085eba493fbde5ea3 100644 (file)
--- a/src/test/regress/expected/numerology.out
+++ b/src/test/regress/expected/numerology.out
@@ -171,7 +171,7 @@ SELECT -0x8000000000000001;
 
 -- error cases
 SELECT 123abc;
-ERROR:  trailing junk after numeric literal at or near "123a"
+ERROR:  trailing junk after numeric literal at or near "123abc"
 LINE 1: SELECT 123abc;
                ^
 SELECT 0x0o;
@@ -318,7 +318,7 @@ ERROR:  trailing junk after numeric literal at or near "100_"
 LINE 1: SELECT 100_;
                ^
 SELECT 100__000;
-ERROR:  trailing junk after numeric literal at or near "100_"
+ERROR:  trailing junk after numeric literal at or near "100__000"
 LINE 1: SELECT 100__000;
                ^
 SELECT _1_000.5;
@@ -330,7 +330,7 @@ ERROR:  trailing junk after numeric literal at or near "1_000_"
 LINE 1: SELECT 1_000_.5;
                ^
 SELECT 1_000._5;
-ERROR:  trailing junk after numeric literal at or near "1_000._"
+ERROR:  trailing junk after numeric literal at or near "1_000._5"
 LINE 1: SELECT 1_000._5;
                ^
 SELECT 1_000.5_;
@@ -338,11 +338,11 @@ ERROR:  trailing junk after numeric literal at or near "1_000.5_"
 LINE 1: SELECT 1_000.5_;
                ^
 SELECT 1_000.5e_1;
-ERROR:  trailing junk after numeric literal at or near "1_000.5e"
+ERROR:  trailing junk after numeric literal at or near "1_000.5e_1"
 LINE 1: SELECT 1_000.5e_1;
                ^
 PREPARE p1 AS SELECT $0_1;
-ERROR:  trailing junk after parameter at or near "$0_"
+ERROR:  trailing junk after parameter at or near "$0_1"
 LINE 1: PREPARE p1 AS SELECT $0_1;
                              ^
 --