Prevent mis-encoding of "trailing junk after numeric literal" errors.

tglsfdc · tglsfdc · commit f37ac613a835 · 2024-09-05T12:42:33.000-04:00
Since commit 2549f06, we reject an identifier immediately following a numeric literal (without separating whitespace), because that risks ambiguity with hex/octal/binary integers. However, that patch used token patterns like "{integer}{ident_start}", which is problematic because {ident_start} matches only a single byte. If the first character after the integer is a multibyte character, this ends up with flex reporting an error message that includes a partial multibyte character. That can cause assorted bad-encoding problems downstream, both in the report to the client and in the postmaster log file. To fix, use {identifier} not {ident_start} in the "junk" token patterns, so that they will match complete multibyte characters. This seems generally better user experience quite aside from the encoding problem: for "123abc" the error message will now say that the error appeared at or near "123abc" instead of "123a". While at it, add some commentary about why these patterns exist and how they work. Report and patch by Karina Litskevich; review by Pavel Borisov. Back-patch to v15 where the problem came in. Discussion: https://postgr.es/m/CACiT8iZ_diop=0zJ7zuY3BXegJpkKK1Av-PU7xh0EDYHsa5+=g@mail.gmail.com
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
@@ -398,12 +398,12 @@ decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail		({integer}|{decimal})[Ee][-+]
 
-integer_junk	{integer}{ident_start}
-decimal_junk	{decimal}{ident_start}
-real_junk		{real}{ident_start}
+integer_junk	{integer}{identifier}
+decimal_junk	{decimal}{identifier}
+real_junk		{real}{identifier}
 
 param			\${integer}
-param_junk		\${integer}{ident_start}
+param_junk		\${integer}{identifier}
 
 other			.
 
diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
@@ -336,12 +336,12 @@ decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail		({integer}|{decimal})[Ee][-+]
 
-integer_junk	{integer}{ident_start}
-decimal_junk	{decimal}{ident_start}
-real_junk		{real}{ident_start}
+integer_junk	{integer}{identifier}
+decimal_junk	{decimal}{identifier}
+real_junk		{real}{identifier}
 
 param			\${integer}
-param_junk		\${integer}{ident_start}
+param_junk		\${integer}{identifier}
 
 /* psql-specific: characters allowed in variable names */
 variable_char	[A-Za-z\200-\377_0-9]
diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l
@@ -369,12 +369,12 @@ decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail		({integer}|{decimal})[Ee][-+]
 
-integer_junk	{integer}{ident_start}
-decimal_junk	{decimal}{ident_start}
-real_junk		{real}{ident_start}
+integer_junk	{integer}{identifier}
+decimal_junk	{decimal}{identifier}
+real_junk		{real}{identifier}
 
 param			\${integer}
-param_junk		\${integer}{ident_start}
+param_junk		\${integer}{identifier}
 
 /* special characters for other dbms */
 /* we have to react differently in compat mode */
diff --git a/src/test/regress/expected/numerology.out b/src/test/regress/expected/numerology.out
@@ -6,15 +6,15 @@
 -- Trailing junk in numeric literals
 --
 SELECT 123abc;
-ERROR:  trailing junk after numeric literal at or near "123a"
+ERROR:  trailing junk after numeric literal at or near "123abc"
 LINE 1: SELECT 123abc;
                ^
 SELECT 0x0o;
-ERROR:  trailing junk after numeric literal at or near "0x"
+ERROR:  trailing junk after numeric literal at or near "0x0o"
 LINE 1: SELECT 0x0o;
                ^
 SELECT 1_2_3;
-ERROR:  trailing junk after numeric literal at or near "1_"
+ERROR:  trailing junk after numeric literal at or near "1_2_3"
 LINE 1: SELECT 1_2_3;
                ^
 SELECT 0.a;

Original file line number	Diff line number	Diff line change
`@@ -6,15 +6,15 @@`
`6`	`6`	`-- Trailing junk in numeric literals`
`7`	`7`	`--`
`8`	`8`	`SELECT 123abc;`
`9`		`-ERROR: trailing junk after numeric literal at or near "123a"`
	`9`	`+ERROR: trailing junk after numeric literal at or near "123abc"`
`10`	`10`	`LINE 1: SELECT 123abc;`
`11`	`11`	`^`
`12`	`12`	`SELECT 0x0o;`
`13`		`-ERROR: trailing junk after numeric literal at or near "0x"`
	`13`	`+ERROR: trailing junk after numeric literal at or near "0x0o"`
`14`	`14`	`LINE 1: SELECT 0x0o;`
`15`	`15`	`^`
`16`	`16`	`SELECT 1_2_3;`
`17`		`-ERROR: trailing junk after numeric literal at or near "1_"`
	`17`	`+ERROR: trailing junk after numeric literal at or near "1_2_3"`
`18`	`18`	`LINE 1: SELECT 1_2_3;`
`19`	`19`	`^`
`20`	`20`	`SELECT 0.a;`