Support National string literal with lower case n (apache#612)

mskrzypkows · Maciej Skrzypkowski · web-flow · commit d4e5b4d5e89f · 2022-09-27T09:07:13.000-04:00
* National string literal with lower case n

It's used by Snowflake

* Corrected DB name

Co-authored-by: Maciej Skrzypkowski &lt;maciej.skrzypkowski@satoricyber.com&gt;
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -387,7 +387,8 @@ impl<'a> Tokenizer<'a> {
                     }
                     Ok(Some(Token::Whitespace(Whitespace::Newline)))
                 }
-                'N' => {
+                // Redshift uses lower case n for national string literal
+                n @ 'N' | n @ 'n' => {
                     chars.next(); // consume, to check the next char
                     match chars.peek() {
                         Some('\'') => {
@@ -397,7 +398,7 @@ impl<'a> Tokenizer<'a> {
                         }
                         _ => {
                             // regular identifier starting with an "N"
-                            let s = self.tokenize_word('N', chars);
+                            let s = self.tokenize_word(n, chars);
                             Ok(Some(Token::make_word(&s, None)))
                         }
                     }
diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs
@@ -2851,6 +2851,7 @@ fn parse_literal_string() {
     );
 
     one_statement_parses_to("SELECT x'deadBEEF'", "SELECT X'deadBEEF'");
+    one_statement_parses_to("SELECT n'national string'", "SELECT N'national string'");
 }
 
 #[test]

Original file line number	Diff line number	Diff line change
`@@ -387,7 +387,8 @@ impl<'a> Tokenizer<'a> {`
`387`	`387`	`}`
`388`	`388`	`Ok(Some(Token::Whitespace(Whitespace::Newline)))`
`389`	`389`	`}`
`390`		`- 'N' => {`
	`390`	`+ // Redshift uses lower case n for national string literal`
	`391`	`+ n @ 'N' \| n @ 'n' => {`
`391`	`392`	`chars.next(); // consume, to check the next char`
`392`	`393`	`match chars.peek() {`
`393`	`394`	`Some('\'') => {`
`@@ -397,7 +398,7 @@ impl<'a> Tokenizer<'a> {`
`397`	`398`	`}`
`398`	`399`	`_ => {`
`399`	`400`	`// regular identifier starting with an "N"`
`400`		`- let s = self.tokenize_word('N', chars);`
	`401`	`+ let s = self.tokenize_word(n, chars);`
`401`	`402`	`Ok(Some(Token::make_word(&s, None)))`
`402`	`403`	`}`
`403`	`404`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2851,6 +2851,7 @@ fn parse_literal_string() {`
`2851`	`2851`	`);`
`2852`	`2852`
`2853`	`2853`	`one_statement_parses_to("SELECT x'deadBEEF'", "SELECT X'deadBEEF'");`
	`2854`	`+ one_statement_parses_to("SELECT n'national string'", "SELECT N'national string'");`
`2854`	`2855`	`}`
`2855`	`2856`
`2856`	`2857`	`#[test]`