Skip to content

Commit 0ea9efb

Browse files
tglsfdctglsfdc
committed
Split psql's lexer into two separate .l files for SQL and backslash cases.
This gets us to a point where psqlscan.l can be used by other frontend programs for the same purpose psql uses it for, ie to detect when it's collected a complete SQL command from input that is divided across line boundaries. Moreover, other programs can supply their own lexers for backslash commands of their own choosing. A follow-on patch will use this in pgbench. The end result here is roughly the same as in Kyotaro Horiguchi's 0001-Make-SQL-parser-part-of-psqlscan-independent-from-ps.patch, although the details of the method for switching between lexers are quite different. Basically, in this patch we share the entire PsqlScanState, YY_BUFFER_STATE stack, *and* yyscan_t between different lexers. The only thing we need to do to switch to a different lexer is to make sure the start_state is valid for the new lexer. This works because flex doesn't keep any other persistent state that depends on the specific lexing tables generated for a particular .l file. (We are assuming that both lexers are built with the same flex version, or at least versions that are compatible with respect to the contents of yyscan_t; but that doesn't seem likely to be a big problem in practice, considering how slowly flex changes.) Aside from being more efficient than Horiguchi-san's original solution, this avoids possible corner-case changes in semantics: the original code was capable of popping the input buffer stack while still staying in backslash-related parsing states. I'm not sure that that equates to any useful user-visible behaviors, but I'm not sure it doesn't either, so I'm loath to assume that we only need to consider the topmost buffer when parsing a backslash command. I've attempted to update the MSVC build scripts for the added .l file, but will rely on the buildfarm to see if I missed anything. Kyotaro Horiguchi and Tom Lane
1 parent 2719905 commit 0ea9efb

12 files changed

+1005
-755
lines changed

‎src/bin/psql/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
/psqlscan.c
2+
/psqlscanslash.c
23
/sql_help.h
34
/sql_help.c
45
/dumputils.c

‎src/bin/psql/Makefile

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) -I$(top_srcdir)/src/bin/p
2323
OBJS= command.o common.o help.o input.o stringutils.o mainloop.o copy.o \
2424
startup.o prompt.o variables.o large_obj.o print.o describe.o \
2525
tab-complete.o mbprint.o dumputils.o keywords.o kwlookup.o \
26-
sql_help.o psqlscan.o \
26+
sql_help.o psqlscan.o psqlscanslash.o \
2727
$(WIN32RES)
2828

2929

@@ -47,12 +47,16 @@ sql_help.h: create_help.pl $(wildcard $(REFDOCDIR)/*.sgml)
4747
psqlscan.c: FLEXFLAGS = -Cfe -p -p
4848
psqlscan.c: FLEX_NO_BACKUP=yes
4949

50-
# Latest flex causes warnings in this file.
50+
psqlscanslash.c: FLEXFLAGS = -Cfe -p -p
51+
psqlscanslash.c: FLEX_NO_BACKUP=yes
52+
53+
# Latest flex causes warnings in these files.
5154
ifeq ($(GCC),yes)
5255
psqlscan.o: CFLAGS += -Wno-error
56+
psqlscanslash.o: CFLAGS += -Wno-error
5357
endif
5458

55-
distprep: sql_help.h psqlscan.c
59+
distprep: sql_help.h psqlscan.c psqlscanslash.c
5660

5761
install: all installdirs
5862
$(INSTALL_PROGRAM) psql$(X) '$(DESTDIR)$(bindir)/psql$(X)'
@@ -64,9 +68,10 @@ installdirs:
6468
uninstall:
6569
rm -f '$(DESTDIR)$(bindir)/psql$(X)' '$(DESTDIR)$(datadir)/psqlrc.sample'
6670

67-
# psqlscan.c is in the distribution tarball, so is not cleaned here
6871
clean distclean:
6972
rm -f psql$(X) $(OBJS) dumputils.c keywords.c kwlookup.c lex.backup
7073

74+
# files removed here are supposed to be in the distribution tarball,
75+
# so do not clean them in the clean/distclean rules
7176
maintainer-clean: distclean
72-
rm -f sql_help.h sql_help.c psqlscan.c
77+
rm -f sql_help.h sql_help.c psqlscan.c psqlscanslash.c

‎src/bin/psql/command.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
#include "large_obj.h"
4646
#include "mainloop.h"
4747
#include "print.h"
48-
#include "psqlscan.h"
48+
#include "psqlscanslash.h"
4949
#include "settings.h"
5050
#include "variables.h"
5151

‎src/bin/psql/nls.mk

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
CATALOG_NAME = psql
33
AVAIL_LANGUAGES = cs de es fr it ja pl pt_BR ru zh_CN zh_TW
44
GETTEXT_FILES = command.c common.c copy.c help.c input.c large_obj.c \
5-
mainloop.c print.c psqlscan.c startup.c describe.c sql_help.h sql_help.c \
5+
mainloop.c print.c psqlscan.c psqlscanslash.c startup.c \
6+
describe.c sql_help.h sql_help.c \
67
tab-complete.c variables.c \
78
../../common/exec.c ../../common/fe_memutils.c ../../common/username.c \
89
../../common/wait_error.c

‎src/bin/psql/psqlscan.h

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,6 @@ typedef enum
2525
PSCAN_EOL /* end of line, SQL possibly complete */
2626
} PsqlScanResult;
2727

28-
/* Different ways for scan_slash_option to handle parameter words */
29-
enum slash_option_type
30-
{
31-
OT_NORMAL, /* normal case */
32-
OT_SQLID, /* treat as SQL identifier */
33-
OT_SQLIDHACK, /* SQL identifier, but don't downcase */
34-
OT_FILEPIPE, /* it's a filename or pipe */
35-
OT_WHOLE_LINE, /* just snarf the rest of the line */
36-
OT_NO_EVAL /* no expansion of backticks or variables */
37-
};
38-
3928
/* Callback functions to be used by the lexer */
4029
typedef struct PsqlScanCallbacks
4130
{
@@ -61,15 +50,8 @@ extern PsqlScanResult psql_scan(PsqlScanState state,
6150

6251
extern void psql_scan_reset(PsqlScanState state);
6352

64-
extern bool psql_scan_in_quote(PsqlScanState state);
65-
66-
extern char *psql_scan_slash_command(PsqlScanState state);
67-
68-
extern char *psql_scan_slash_option(PsqlScanState state,
69-
enum slash_option_type type,
70-
char *quote,
71-
bool semicolon);
53+
extern void psql_scan_reselect_sql_lexer(PsqlScanState state);
7254

73-
extern void psql_scan_slash_command_end(PsqlScanState state);
55+
extern bool psql_scan_in_quote(PsqlScanState state);
7456

7557
#endif /* PSQLSCAN_H */

0 commit comments

Comments
 (0)