Remove plpgsql's separate lexer (finally!), in favor of using the core lexer

directly.  This was a lot of trouble, but should be worth it in terms of
not having to keep the plpgsql lexer in step with core anymore.  In addition
the handling of keywords is significantly better-structured, allowing us to
de-reserve a number of words that plpgsql formerly treated as reserved.

Author: tglsfdc

src/backend/parser/scan.l

@@ -24,7 +24,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/scan.l,v 1.163 2009/11/09 18:38:48 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/scan.l,v 1.164 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -85,6 +85,7 @@ static void addlitchar(unsigned char ychar, core_yyscan_t yyscanner);
 static char *litbufdup(core_yyscan_t yyscanner);
 static char *litbuf_udeescape(unsigned char escape, core_yyscan_t yyscanner);
 static unsigned char unescape_single_char(unsigned char c, core_yyscan_t yyscanner);
+static int	process_integer_literal(const char *token, YYSTYPE *lval);
 static bool is_utf16_surrogate_first(pg_wchar c);
 static bool is_utf16_surrogate_second(pg_wchar c);
 static pg_wchar surrogate_pair_to_codepoint(pg_wchar first, pg_wchar second);
@@ -339,12 +340,15 @@ operator		{op_chars}+
  * instead we pass it separately to parser. there it gets
  * coerced via doNegate() -- Leon aug 20 1999
  *
+* {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot, 10.
+*
  * {realfail1} and {realfail2} are added to prevent the need for scanner
  * backup when the {real} rule fails to match completely.
  */
 
 integer			{digit}+
 decimal			(({digit}*\.{digit}+)|({digit}+\.{digit}*))
+decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail1		({integer}|{decimal})[Ee]
 realfail2		({integer}|{decimal})[Ee][-+]
@@ -846,31 +850,20 @@ other			.
 				}
 
 {integer}		{
-					long val;
-					char* endptr;
-
 					SET_YYLLOC();
-					errno = 0;
-					val = strtol(yytext, &endptr, 10);
-					if (*endptr != '\0' || errno == ERANGE
-#ifdef HAVE_LONG_INT_64
-						/* if long > 32 bits, check for overflow of int4 */
-						|| val != (long) ((int32) val)
-#endif
-						)
-					{
-						/* integer too large, treat it as a float */
-						yylval->str = pstrdup(yytext);
-						return FCONST;
-					}
-					yylval->ival = val;
-					return ICONST;
+					return process_integer_literal(yytext, yylval);
 				}
 {decimal}		{
 					SET_YYLLOC();
 					yylval->str = pstrdup(yytext);
 					return FCONST;
 				}
+{decimalfail}	{
+					/* throw back the .., and treat as integer */
+					yyless(yyleng-2);
+					SET_YYLLOC();
+					return process_integer_literal(yytext, yylval);
+				}
 {real}			{
 					SET_YYLLOC();
 					yylval->str = pstrdup(yytext);
@@ -1121,6 +1114,29 @@ litbufdup(core_yyscan_t yyscanner)
 	return new;
 }
 
+static int
+process_integer_literal(const char *token, YYSTYPE *lval)
+{
+	long		val;
+	char	   *endptr;
+
+	errno = 0;
+	val = strtol(token, &endptr, 10);
+	if (*endptr != '\0' || errno == ERANGE
+#ifdef HAVE_LONG_INT_64
+		/* if long > 32 bits, check for overflow of int4 */
+		|| val != (long) ((int32) val)
+#endif
+		)
+	{
+		/* integer too large, treat it as a float */
+		lval->str = pstrdup(token);
+		return FCONST;
+	}
+	lval->ival = val;
+	return ICONST;
+}
+
 static int
 hexval(unsigned char c)
 {

src/backend/parser/scansup.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.37 2009/01/01 17:23:46 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.38 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -197,7 +197,6 @@ bool
 scanner_isspace(char ch)
 {
 	/* This must match scan.l's list of {space} characters */
-	/* and plpgsql's scan.l as well */
 	if (ch == ' ' ||
 		ch == '\t' ||
 		ch == '\n' ||

src/bin/psql/psqlscan.l

@@ -33,7 +33,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/bin/psql/psqlscan.l,v 1.29 2009/09/27 03:27:24 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/bin/psql/psqlscan.l,v 1.30 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -364,12 +364,15 @@ operator		{op_chars}+
  * instead we pass it separately to parser. there it gets
  * coerced via doNegate() -- Leon aug 20 1999
  *
+ * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot, 10.
+ *
  * {realfail1} and {realfail2} are added to prevent the need for scanner
  * backup when the {real} rule fails to match completely.
  */
 
 integer			{digit}+
 decimal			(({digit}*\.{digit}+)|({digit}+\.{digit}*))
+decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail1		({integer}|{decimal})[Ee]
 realfail2		({integer}|{decimal})[Ee][-+]
@@ -776,6 +779,11 @@ other			.
 {decimal}		{
 					ECHO;
 				}
+{decimalfail}	{
+					/* throw back the .., and treat as integer */
+					yyless(yyleng-2);
+					ECHO;
+				}
 {real}			{
 					ECHO;
 				}

src/pl/plpgsql/src/.cvsignore

@@ -1,3 +1,2 @@
 pl_gram.c
 pl_gram.h
-pl_scan.c

src/pl/plpgsql/src/Makefile

@@ -2,7 +2,7 @@
 #
 # Makefile for the plpgsql shared object
 #
-# $PostgreSQL: pgsql/src/pl/plpgsql/src/Makefile,v 1.34 2009/08/28 20:26:19 petere Exp $
+# $PostgreSQL: pgsql/src/pl/plpgsql/src/Makefile,v 1.35 2009/11/12 00:13:00 tgl Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -17,7 +17,7 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 SHLIB_LINK = $(filter -lintl, $(LIBS))
 rpath =
 
-OBJS = pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o
+OBJS = pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o pl_scanner.o
 
 all: all-lib
 
@@ -33,10 +33,7 @@ uninstall: uninstall-lib
 
 
 # Force these dependencies to be known even without dependency info built:
-pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o: plpgsql.h pl_gram.h
-
-# pl_scan is compiled as part of pl_gram
-pl_gram.o: pl_scan.c
+pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o pl_scanner.o: plpgsql.h pl_gram.h
 
 # See notes in src/backend/parser/Makefile about the following two rules
 
@@ -49,23 +46,12 @@ else
 	@$(missing) bison $< $@
 endif
 
-# Because we use %option case-insensitive, flex's results could vary
-# depending on what the compile-time locale setting is.  Hence, force
-# it to see LC_CTYPE=C to ensure consistent build results.
-
-pl_scan.c: scan.l
-ifdef FLEX
-	LC_CTYPE=C $(FLEX) $(FLEXFLAGS) -o'$@' $<
-else
-	@$(missing) flex $< $@
-endif
-
-distprep: pl_scan.c pl_gram.h pl_gram.c
+distprep: pl_gram.h pl_gram.c
 
-# pl_gram.c, pl_gram.h, and pl_scan.c are in the distribution tarball,
+# pl_gram.c and pl_gram.h are in the distribution tarball,
 # so they are not cleaned here.
 clean distclean: clean-lib
 	rm -f $(OBJS)
 
 maintainer-clean: clean
-	rm -f pl_gram.c pl_gram.h pl_scan.c
+	rm -f pl_gram.c pl_gram.h