Skip to content

Commit ae6d06f

Browse files
michaelpqmichaelpq
committed
Handle \v as a whitespace character in parsers
This commit comes as a continuation of the discussion that has led to d522b05, as \v was handled inconsistently when parsing array values or anything going through the parsers, and changing a parser behavior in stable branches is a scary thing to do. The parsing of array values now uses the more central scanner_isspace() and array_isspace() is removed. As pointing out by Peter Eisentraut, fix a confusing reference to horizontal space in the parsers with the term "horiz_space". \f was included in this set since 3cfdd8f from 2000, but it is not horizontal. "horiz_space" is renamed to "non_newline_space", to refer to all whitespace characters except newlines. The changes impact the parsers for the backend, psql, seg, cube, ecpg and replication commands. Note that JSON should not escape \v, as per RFC 7159, so these are not touched. Reviewed-by: Peter Eisentraut, Tom Lane Discussion: https://postgr.es/m/ZJKcjNwWHHvw9ksQ@paquier.xyz
1 parent 4f4d734 commit ae6d06f

File tree

13 files changed

+77
-56
lines changed

13 files changed

+77
-56
lines changed

‎contrib/cube/cubescan.l

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ NaN [nN][aA][nN]
6363
\( cube_yylval = "("; return O_PAREN;
6464
\) cube_yylval = ")"; return C_PAREN;
6565
\, cube_yylval = ","; return COMMA;
66-
[ \t\n\r\f]+ /* discard spaces */
66+
[ \t\n\r\f\v]+ /* discard spaces */
6767
. return yytext[0]; /* alert parser of the garbage */
6868

6969
%%

‎contrib/hstore/expected/hstore_utf8.out

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,34 @@ SELECT 'keyąfoo=>valueą'::hstore;
3434
"keyąfoo"=>"valueą"
3535
(1 row)
3636

37+
-- More patterns that may depend on isspace() and locales, all discarded.
38+
SELECT E'key\u000A=>value\u000A'::hstore; -- \n
39+
hstore
40+
----------------
41+
"key"=>"value"
42+
(1 row)
43+
44+
SELECT E'key\u0009=>value\u0009'::hstore; -- \t
45+
hstore
46+
----------------
47+
"key"=>"value"
48+
(1 row)
49+
50+
SELECT E'key\u000D=>value\u000D'::hstore; -- \r
51+
hstore
52+
----------------
53+
"key"=>"value"
54+
(1 row)
55+
56+
SELECT E'key\u000B=>value\u000B'::hstore; -- \v
57+
hstore
58+
----------------
59+
"key"=>"value"
60+
(1 row)
61+
62+
SELECT E'key\u000C=>value\u000C'::hstore; -- \f
63+
hstore
64+
----------------
65+
"key"=>"value"
66+
(1 row)
67+

‎contrib/hstore/sql/hstore_utf8.sql

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,10 @@ SELECT E'key\u0105=>value\u0105'::hstore;
1717
SELECT 'keyą=>valueą'::hstore;
1818
SELECT 'ą=>ą'::hstore;
1919
SELECT 'keyąfoo=>valueą'::hstore;
20+
21+
-- More patterns that may depend on isspace() and locales, all discarded.
22+
SELECT E'key\u000A=>value\u000A'::hstore; -- \n
23+
SELECT E'key\u0009=>value\u0009'::hstore; -- \t
24+
SELECT E'key\u000D=>value\u000D'::hstore; -- \r
25+
SELECT E'key\u000B=>value\u000B'::hstore; -- \v
26+
SELECT E'key\u000C=>value\u000C'::hstore; -- \f

‎contrib/seg/segscan.l

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ float ({integer}|{real})([eE]{integer})?
5959
\< seg_yylval.text = "<"; return EXTENSION;
6060
\> seg_yylval.text = ">"; return EXTENSION;
6161
\~ seg_yylval.text = "~"; return EXTENSION;
62-
[ \t\n\r\f]+ /* discard spaces */
62+
[ \t\n\r\f\v]+ /* discard spaces */
6363
. return yytext[0]; /* alert parser of the garbage */
6464

6565
%%

‎src/backend/parser/parse_type.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -742,7 +742,7 @@ typeStringToTypeName(const char *str, Node *escontext)
742742
ErrorContextCallback ptserrcontext;
743743

744744
/* make sure we give useful error for empty input */
745-
if (strspn(str, " \t\n\r\f") == strlen(str))
745+
if (strspn(str, " \t\n\r\f\v") == strlen(str))
746746
goto fail;
747747

748748
/*

‎src/backend/parser/scan.l

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -213,16 +213,16 @@ extern void core_yyset_column(int column_no, yyscan_t yyscanner);
213213
* versions of Postgres failed to recognize -- as a comment if the input
214214
* did not end with a newline.
215215
*
216-
* XXX perhaps \f (formfeed) should be treated as a newline as well?
216+
* non_newline_space tracks all the other space characters except newlines.
217217
*
218218
* XXX if you change the set of whitespace characters, fix scanner_isspace()
219219
* to agree.
220220
*/
221221

222-
space [ \t\n\r\f]
223-
horiz_space [ \t\f]
224-
newline [\n\r]
225-
non_newline [^\n\r]
222+
space [ \t\n\r\f\v]
223+
non_newline_space [ \t\f\v]
224+
newline [\n\r]
225+
non_newline [^\n\r]
226226

227227
comment ("--"{non_newline}*)
228228

@@ -236,8 +236,8 @@ whitespace ({space}+|{comment})
236236
*/
237237

238238
special_whitespace ({space}+|{comment}{newline})
239-
horiz_whitespace ({horiz_space}|{comment})
240-
whitespace_with_newline ({horiz_whitespace}*{newline}{special_whitespace}*)
239+
non_newline_whitespace ({non_newline_space}|{comment})
240+
whitespace_with_newline ({non_newline_whitespace}*{newline}{special_whitespace}*)
241241

242242
quote '
243243
/* If we see {quote} then {quotecontinue}, the quoted string continues */
@@ -1414,6 +1414,8 @@ unescape_single_char(unsigned char c, core_yyscan_t yyscanner)
14141414
return '\r';
14151415
case 't':
14161416
return '\t';
1417+
case 'v':
1418+
return '\v';
14171419
default:
14181420
/* check for backslash followed by non-7-bit-ASCII */
14191421
if (c == '\0' || IS_HIGHBIT_SET(c))

‎src/backend/parser/scansup.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ scanner_isspace(char ch)
121121
ch == '\t' ||
122122
ch == '\n' ||
123123
ch == '\r' ||
124+
ch == '\v' ||
124125
ch == '\f')
125126
return true;
126127
return false;

‎src/backend/replication/repl_scanner.l

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ static void addlitchar(unsigned char ychar);
7373
%x xd
7474
%x xq
7575

76-
space [ \t\n\r\f]
76+
space [ \t\n\r\f\v]
7777

7878
quote '
7979
quotestop {quote}

‎src/backend/utils/adt/arrayfuncs.c

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "nodes/nodeFuncs.h"
2525
#include "nodes/supportnodes.h"
2626
#include "optimizer/optimizer.h"
27+
#include "parser/scansup.h"
2728
#include "port/pg_bitutils.h"
2829
#include "utils/array.h"
2930
#include "utils/arrayaccess.h"
@@ -89,7 +90,6 @@ typedef struct ArrayIteratorData
8990
int current_item; /* the item # we're at in the array */
9091
} ArrayIteratorData;
9192

92-
static bool array_isspace(char ch);
9393
static int ArrayCount(const char *str, int *dim, char typdelim,
9494
Node *escontext);
9595
static bool ReadArrayStr(char *arrayStr, const char *origStr,
@@ -254,7 +254,7 @@ array_in(PG_FUNCTION_ARGS)
254254
* Note: we currently allow whitespace between, but not within,
255255
* dimension items.
256256
*/
257-
while (array_isspace(*p))
257+
while (scanner_isspace(*p))
258258
p++;
259259
if (*p != '[')
260260
break; /* no more dimension items */
@@ -338,7 +338,7 @@ array_in(PG_FUNCTION_ARGS)
338338
errdetail("Missing \"%s\" after array dimensions.",
339339
ASSGN)));
340340
p += strlen(ASSGN);
341-
while (array_isspace(*p))
341+
while (scanner_isspace(*p))
342342
p++;
343343

344344
/*
@@ -434,27 +434,6 @@ array_in(PG_FUNCTION_ARGS)
434434
PG_RETURN_ARRAYTYPE_P(retval);
435435
}
436436

437-
/*
438-
* array_isspace() --- a non-locale-dependent isspace()
439-
*
440-
* We used to use isspace() for parsing array values, but that has
441-
* undesirable results: an array value might be silently interpreted
442-
* differently depending on the locale setting. Now we just hard-wire
443-
* the traditional ASCII definition of isspace().
444-
*/
445-
static bool
446-
array_isspace(char ch)
447-
{
448-
if (ch == ' ' ||
449-
ch == '\t' ||
450-
ch == '\n' ||
451-
ch == '\r' ||
452-
ch == '\v' ||
453-
ch == '\f')
454-
return true;
455-
return false;
456-
}
457-
458437
/*
459438
* ArrayCount
460439
* Determines the dimensions for an array string.
@@ -654,7 +633,7 @@ ArrayCount(const char *str, int *dim, char typdelim, Node *escontext)
654633
itemdone = true;
655634
nelems[nest_level - 1]++;
656635
}
657-
else if (!array_isspace(*ptr))
636+
else if (!scanner_isspace(*ptr))
658637
{
659638
/*
660639
* Other non-space characters must be after a
@@ -684,7 +663,7 @@ ArrayCount(const char *str, int *dim, char typdelim, Node *escontext)
684663
/* only whitespace is allowed after the closing brace */
685664
while (*ptr)
686665
{
687-
if (!array_isspace(*ptr++))
666+
if (!scanner_isspace(*ptr++))
688667
ereturn(escontext, -1,
689668
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
690669
errmsg("malformed array literal: \"%s\"", str),
@@ -884,7 +863,7 @@ ReadArrayStr(char *arrayStr,
884863
indx[ndim - 1]++;
885864
srcptr++;
886865
}
887-
else if (array_isspace(*srcptr))
866+
else if (scanner_isspace(*srcptr))
888867
{
889868
/*
890869
* If leading space, drop it immediately. Else, copy
@@ -1176,7 +1155,7 @@ array_out(PG_FUNCTION_ARGS)
11761155
overall_length += 1;
11771156
}
11781157
else if (ch == '{' || ch == '}' || ch == typdelim ||
1179-
array_isspace(ch))
1158+
scanner_isspace(ch))
11801159
needquote = true;
11811160
}
11821161
}

‎src/bin/psql/psqlscanslash.l

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ extern void slash_yyset_column(int column_no, yyscan_t yyscanner);
108108
/*
109109
* Assorted character class definitions that should match psqlscan.l.
110110
*/
111-
space [ \t\n\r\f]
111+
space [ \t\n\r\f\v]
112112
quote '
113113
xeoctesc [\\][0-7]{1,3}
114114
xehexesc [\\]x[0-9A-Fa-f]{1,2}

‎src/fe_utils/psqlscan.l

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -149,16 +149,16 @@ extern void psql_yyset_column(int column_no, yyscan_t yyscanner);
149149
* versions of Postgres failed to recognize -- as a comment if the input
150150
* did not end with a newline.
151151
*
152-
* XXX perhaps \f (formfeed) should be treated as a newline as well?
152+
* non_newline_space tracks all space characters except newlines.
153153
*
154154
* XXX if you change the set of whitespace characters, fix scanner_isspace()
155155
* to agree.
156156
*/
157157

158-
space [ \t\n\r\f]
159-
horiz_space [ \t\f]
160-
newline [\n\r]
161-
non_newline [^\n\r]
158+
space [ \t\n\r\f\v]
159+
non_newline_space [ \t\f\v]
160+
newline [\n\r]
161+
non_newline [^\n\r]
162162

163163
comment ("--"{non_newline}*)
164164

@@ -172,8 +172,8 @@ whitespace ({space}+|{comment})
172172
*/
173173

174174
special_whitespace ({space}+|{comment}{newline})
175-
horiz_whitespace ({horiz_space}|{comment})
176-
whitespace_with_newline ({horiz_whitespace}*{newline}{special_whitespace}*)
175+
non_newline_whitespace ({non_newline_space}|{comment})
176+
whitespace_with_newline ({non_newline_whitespace}*{newline}{special_whitespace}*)
177177

178178
quote '
179179
/* If we see {quote} then {quotecontinue}, the quoted string continues */

‎src/fe_utils/string_utils.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ appendPGArray(PQExpBuffer buffer, const char *value)
761761

762762
if (ch == '"' || ch == '\\' ||
763763
ch == '{' || ch == '}' || ch == ',' ||
764-
/* these match array_isspace(): */
764+
/* these match scanner_isspace(): */
765765
ch == ' ' || ch == '\t' || ch == '\n' ||
766766
ch == '\r' || ch == '\v' || ch == '\f')
767767
{

‎src/interfaces/ecpg/preproc/pgc.l

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -180,16 +180,16 @@ static struct _if_value
180180
* versions of Postgres failed to recognize -- as a comment if the input
181181
* did not end with a newline.
182182
*
183-
* XXX perhaps \f (formfeed) should be treated as a newline as well?
183+
* non_newline_space tracks all space characters except newlines.
184184
*
185185
* XXX if you change the set of whitespace characters, fix ecpg_isspace()
186186
* to agree.
187187
*/
188188

189-
space [ \t\n\r\f]
190-
horiz_space [ \t\f]
191-
newline [\n\r]
192-
non_newline [^\n\r]
189+
space [ \t\n\r\f\v]
190+
non_newline_space [ \t\f\v]
191+
newline [\n\r]
192+
non_newline [^\n\r]
193193

194194
comment ("--"{non_newline}*)
195195

@@ -202,8 +202,8 @@ whitespace ({space}+|{comment})
202202
* it, whereas {whitespace} should generally have a * after it...
203203
*/
204204

205-
horiz_whitespace ({horiz_space}|{comment})
206-
whitespace_with_newline ({horiz_whitespace}*{newline}{whitespace}*)
205+
non_newline_whitespace ({non_newline_space}|{comment})
206+
whitespace_with_newline ({non_newline_whitespace}*{newline}{whitespace}*)
207207

208208
quote '
209209
/* If we see {quote} then {quotecontinue}, the quoted string continues */
@@ -1721,7 +1721,8 @@ ecpg_isspace(char ch)
17211721
ch == '\t' ||
17221722
ch == '\n' ||
17231723
ch == '\r' ||
1724-
ch == '\f')
1724+
ch == '\f' ||
1725+
ch == '\v')
17251726
return true;
17261727
return false;
17271728
}

0 commit comments

Comments
 (0)