Merge pull request apache#36 from nickolay/strings

andygrove · web-flow · commit 73f55fe1bb31 · 2019-03-08T06:14:59.000-07:00
Clean up string-related variants in Token and Value
diff --git a/src/sqlast/mod.rs b/src/sqlast/mod.rs
@@ -103,7 +103,7 @@ pub enum ASTNode {
         /// COLUMNS
         columns: Vec<String>,
         /// VALUES a vector of values to be copied
-        values: Vec<Value>,
+        values: Vec<Option<String>>,
     },
     /// UPDATE
     SQLUpdate {
@@ -290,7 +290,7 @@ impl ToString for ASTNode {
                         "\n{}",
                         values
                             .iter()
-                            .map(|v| v.to_string())
+                            .map(|v| v.clone().unwrap_or("\\N".to_string()))
                             .collect::<Vec<String>>()
                             .join("\t")
                     );
diff --git a/src/sqlast/value.rs b/src/sqlast/value.rs
@@ -2,21 +2,17 @@ use chrono::{offset::FixedOffset, DateTime, NaiveDate, NaiveDateTime, NaiveTime}
 
 use uuid::Uuid;
 
-/// SQL values such as int, double, string timestamp
+/// SQL values such as int, double, string, timestamp
 #[derive(Debug, Clone, PartialEq)]
 pub enum Value {
     /// Literal signed long
     Long(i64),
     /// Literal floating point value
     Double(f64),
-    /// Unquoted string
-    String(String),
     /// Uuid value
     Uuid(Uuid),
     /// 'string value'
     SingleQuotedString(String),
-    /// "string value"
-    DoubleQuotedString(String),
     /// Boolean value true or false,
     Boolean(bool),
     /// Date value
@@ -36,10 +32,8 @@ impl ToString for Value {
         match self {
             Value::Long(v) => v.to_string(),
             Value::Double(v) => v.to_string(),
-            Value::String(v) => v.to_string(),
             Value::Uuid(v) => v.to_string(),
             Value::SingleQuotedString(v) => format!("'{}'", v),
-            Value::DoubleQuotedString(v) => format!("\"{}\"", v),
             Value::Boolean(v) => v.to_string(),
             Value::Date(v) => v.to_string(),
             Value::Time(v) => v.to_string(),
diff --git a/src/sqlparser.rs b/src/sqlparser.rs
@@ -157,10 +157,7 @@ impl Parser {
                         }
                     }
                 }
-                Token::Number(_)
-                | Token::String(_)
-                | Token::SingleQuotedString(_)
-                | Token::DoubleQuotedString(_) => {
+                Token::Number(_) | Token::SingleQuotedString(_) => {
                     self.prev_token();
                     self.parse_sql_value()
                 }
@@ -693,7 +690,7 @@ impl Parser {
 
     /// Parse a tab separated values in
     /// COPY payload
-    fn parse_tsv(&mut self) -> Result<Vec<Value>, ParserError> {
+    fn parse_tsv(&mut self) -> Result<Vec<Option<String>>, ParserError> {
         let values = self.parse_tab_value()?;
         Ok(values)
     }
@@ -702,17 +699,17 @@ impl Parser {
         Ok(ASTNode::SQLValue(self.parse_value()?))
     }
 
-    fn parse_tab_value(&mut self) -> Result<Vec<Value>, ParserError> {
+    fn parse_tab_value(&mut self) -> Result<Vec<Option<String>>, ParserError> {
         let mut values = vec![];
         let mut content = String::from("");
         while let Some(t) = self.next_token_no_skip() {
             match t {
                 Token::Whitespace(Whitespace::Tab) => {
-                    values.push(Value::String(content.to_string()));
+                    values.push(Some(content.to_string()));
                     content.clear();
                 }
                 Token::Whitespace(Whitespace::Newline) => {
-                    values.push(Value::String(content.to_string()));
+                    values.push(Some(content.to_string()));
                     content.clear();
                 }
                 Token::Backslash => {
@@ -721,7 +718,7 @@ impl Parser {
                     }
                     if let Some(token) = self.next_token() {
                         if token == Token::Identifier("N".to_string()) {
-                            values.push(Value::Null);
+                            values.push(None);
                         }
                     } else {
                         continue;
@@ -735,6 +732,7 @@ impl Parser {
         Ok(values)
     }
 
+    /// Parse a literal value (numbers, strings, date/time, booleans)
     fn parse_value(&mut self) -> Result<Value, ParserError> {
         match self.next_token() {
             Some(t) => {
@@ -754,14 +752,9 @@ impl Parser {
                         Ok(n) => Ok(Value::Long(n)),
                         Err(e) => parser_err!(format!("Could not parse '{}' as i64: {}", n, e)),
                     },
-                    Token::Identifier(id) => Ok(Value::String(id.to_string())),
-                    Token::String(ref s) => Ok(Value::String(s.to_string())),
                     Token::SingleQuotedString(ref s) => {
                         Ok(Value::SingleQuotedString(s.to_string()))
                     }
-                    Token::DoubleQuotedString(ref s) => {
-                        Ok(Value::DoubleQuotedString(s.to_string()))
-                    }
                     _ => parser_err!(format!("Unsupported value: {:?}", self.peek_token())),
                 }
             }
@@ -792,9 +785,7 @@ impl Parser {
     /// Parse a literal string
     pub fn parse_literal_string(&mut self) -> Result<String, ParserError> {
         match self.next_token() {
-            Some(Token::String(ref s)) => Ok(s.clone()),
             Some(Token::SingleQuotedString(ref s)) => Ok(s.clone()),
-            Some(Token::DoubleQuotedString(ref s)) => Ok(s.clone()),
             other => parser_err!(format!("Expected literal string, found {:?}", other)),
         }
     }
diff --git a/src/sqltokenizer.rs b/src/sqltokenizer.rs
@@ -32,8 +32,7 @@ pub enum Token {
     Keyword(String),
     /// Numeric literal
     Number(String),
-    /// String literal
-    String(String),
+    /// A character that could not be tokenized
     Char(char),
     /// Single quoted string: i.e: 'string'
     SingleQuotedString(String),
@@ -97,7 +96,6 @@ impl ToString for Token {
             Token::Identifier(ref id) => id.to_string(),
             Token::Keyword(ref k) => k.to_string(),
             Token::Number(ref n) => n.to_string(),
-            Token::String(ref s) => s.to_string(),
             Token::Char(ref c) => c.to_string(),
             Token::SingleQuotedString(ref s) => format!("'{}'", s),
             Token::DoubleQuotedString(ref s) => format!("\"{}\"", s),
@@ -194,7 +192,6 @@ impl<'a> Tokenizer<'a> {
                 Token::Identifier(s) => self.col += s.len() as u64,
                 Token::Keyword(s) => self.col += s.len() as u64,
                 Token::Number(s) => self.col += s.len() as u64,
-                Token::String(s) => self.col += s.len() as u64,
                 Token::SingleQuotedString(s) => self.col += s.len() as u64,
                 Token::DoubleQuotedString(s) => self.col += s.len() as u64,
                 _ => self.col += 1,

Original file line number	Diff line number	Diff line change
`@@ -157,10 +157,7 @@ impl Parser {`
`157`	`157`	`}`
`158`	`158`	`}`
`159`	`159`	`}`
`160`		`- Token::Number(_)`
`161`		`- \| Token::String(_)`
`162`		`- \| Token::SingleQuotedString(_)`
`163`		`- \| Token::DoubleQuotedString(_) => {`
	`160`	`+ Token::Number(_) \| Token::SingleQuotedString(_) => {`
`164`	`161`	`self.prev_token();`
`165`	`162`	`self.parse_sql_value()`
`166`	`163`	`}`
`@@ -693,7 +690,7 @@ impl Parser {`
`693`	`690`
`694`	`691`	`/// Parse a tab separated values in`
`695`	`692`	`/// COPY payload`
`696`		`- fn parse_tsv(&mut self) -> Result<Vec<Value>, ParserError> {`
	`693`	`+ fn parse_tsv(&mut self) -> Result<Vec<Option<String>>, ParserError> {`
`697`	`694`	`let values = self.parse_tab_value()?;`
`698`	`695`	`Ok(values)`
`699`	`696`	`}`
`@@ -702,17 +699,17 @@ impl Parser {`
`702`	`699`	`Ok(ASTNode::SQLValue(self.parse_value()?))`
`703`	`700`	`}`
`704`	`701`
`705`		`- fn parse_tab_value(&mut self) -> Result<Vec<Value>, ParserError> {`
	`702`	`+ fn parse_tab_value(&mut self) -> Result<Vec<Option<String>>, ParserError> {`
`706`	`703`	`let mut values = vec![];`
`707`	`704`	`let mut content = String::from("");`
`708`	`705`	`while let Some(t) = self.next_token_no_skip() {`
`709`	`706`	`match t {`
`710`	`707`	`Token::Whitespace(Whitespace::Tab) => {`
`711`		`- values.push(Value::String(content.to_string()));`
	`708`	`+ values.push(Some(content.to_string()));`
`712`	`709`	`content.clear();`
`713`	`710`	`}`
`714`	`711`	`Token::Whitespace(Whitespace::Newline) => {`
`715`		`- values.push(Value::String(content.to_string()));`
	`712`	`+ values.push(Some(content.to_string()));`
`716`	`713`	`content.clear();`
`717`	`714`	`}`
`718`	`715`	`Token::Backslash => {`
`@@ -721,7 +718,7 @@ impl Parser {`
`721`	`718`	`}`
`722`	`719`	`if let Some(token) = self.next_token() {`
`723`	`720`	`if token == Token::Identifier("N".to_string()) {`
`724`		`- values.push(Value::Null);`
	`721`	`+ values.push(None);`
`725`	`722`	`}`
`726`	`723`	`} else {`
`727`	`724`	`continue;`
`@@ -735,6 +732,7 @@ impl Parser {`
`735`	`732`	`Ok(values)`
`736`	`733`	`}`
`737`	`734`
	`735`	`+ /// Parse a literal value (numbers, strings, date/time, booleans)`
`738`	`736`	`fn parse_value(&mut self) -> Result<Value, ParserError> {`
`739`	`737`	`match self.next_token() {`
`740`	`738`	`Some(t) => {`
`@@ -754,14 +752,9 @@ impl Parser {`
`754`	`752`	`Ok(n) => Ok(Value::Long(n)),`
`755`	`753`	`Err(e) => parser_err!(format!("Could not parse '{}' as i64: {}", n, e)),`
`756`	`754`	`},`
`757`		`- Token::Identifier(id) => Ok(Value::String(id.to_string())),`
`758`		`- Token::String(ref s) => Ok(Value::String(s.to_string())),`
`759`	`755`	`Token::SingleQuotedString(ref s) => {`
`760`	`756`	`Ok(Value::SingleQuotedString(s.to_string()))`
`761`	`757`	`}`
`762`		`- Token::DoubleQuotedString(ref s) => {`
`763`		`- Ok(Value::DoubleQuotedString(s.to_string()))`
`764`		`- }`
`765`	`758`	`_ => parser_err!(format!("Unsupported value: {:?}", self.peek_token())),`
`766`	`759`	`}`
`767`	`760`	`}`
`@@ -792,9 +785,7 @@ impl Parser {`
`792`	`785`	`/// Parse a literal string`
`793`	`786`	`pub fn parse_literal_string(&mut self) -> Result<String, ParserError> {`
`794`	`787`	`match self.next_token() {`
`795`		`- Some(Token::String(ref s)) => Ok(s.clone()),`
`796`	`788`	`Some(Token::SingleQuotedString(ref s)) => Ok(s.clone()),`
`797`		`- Some(Token::DoubleQuotedString(ref s)) => Ok(s.clone()),`
`798`	`789`	`other => parser_err!(format!("Expected literal string, found {:?}", other)),`
`799`	`790`	`}`
`800`	`791`	`}`