From 861d0bebe9c4d6c62492585c02b026e618abbdfd Mon Sep 17 00:00:00 2001 From: imotai Date: Mon, 21 Mar 2022 14:06:23 +0800 Subject: [PATCH 01/12] fix: support create database --- src/parser.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parser.rs b/src/parser.rs index 3389ab8d1..b94796520 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1521,6 +1521,8 @@ impl<'a> Parser<'a> { self.parse_create_virtual_table() } else if self.parse_keyword(Keyword::SCHEMA) { self.parse_create_schema() + } else if self.parse_keyword(Keyword::DATABASE) { + self.parse_create_database() } else { self.expected("an object type after CREATE", self.peek_token()) } From 4cdfb53def40b8c53f53fdc7690d77465274ef28 Mon Sep 17 00:00:00 2001 From: imotai Date: Mon, 21 Mar 2022 14:17:46 +0800 Subject: [PATCH 02/12] fix: support create db --- src/parser.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parser.rs b/src/parser.rs index 6ccd4980b..d2bffce9d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1512,6 +1512,8 @@ impl<'a> Parser<'a> { self.parse_create_virtual_table() } else if self.parse_keyword(Keyword::SCHEMA) { self.parse_create_schema() + } else if self.parse_keyword(Keyword::DATABASE) { + self.parse_create_database() } else { self.expected("an object type after CREATE", self.peek_token()) } From 9a04d232c7c0a19090a0ffa3395e1bf8fb1877a3 Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 30 Mar 2022 15:28:05 +0800 Subject: [PATCH 03/12] feat: support mysql charset --- src/ast/data_type.rs | 10 +++++----- src/ast/mod.rs | 5 ++++- src/parser.rs | 28 +++++++++++++++++++++++----- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index 14f769640..602d17dd8 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -25,10 +25,10 @@ use super::value::escape_single_quote_string; #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum DataType { - /// Fixed-length character type e.g. CHAR(10) - Char(Option), + /// Fixed-length character type e.g. CHAR(10), CHAR CHARACTER set utf8mb4 + Char(Option, Option), /// Variable-length character type e.g. VARCHAR(10) - Varchar(Option), + Varchar(Option, Option), /// Uuid type Uuid, /// Large character object e.g. CLOB(1000) @@ -94,8 +94,8 @@ pub enum DataType { impl fmt::Display for DataType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - DataType::Char(size) => format_type_with_optional_length(f, "CHAR", size, false), - DataType::Varchar(size) => { + DataType::Char(size, _) => format_type_with_optional_length(f, "CHAR", size, false), + DataType::Varchar(size, _) => { format_type_with_optional_length(f, "CHARACTER VARYING", size, false) } DataType::Uuid => write!(f, "UUID"), diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 7c6f28ee3..a31b29a45 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -240,6 +240,7 @@ pub enum Expr { Cast { expr: Box, data_type: DataType, + collation: Option, }, /// TRY_CAST an expression to a different data type e.g. `TRY_CAST(foo AS VARCHAR(123))` // this differs from CAST in the choice of how to implement invalid conversions @@ -396,7 +397,9 @@ impl fmt::Display for Expr { write!(f, "{} {}", op, expr) } } - Expr::Cast { expr, data_type } => write!(f, "CAST({} AS {})", expr, data_type), + Expr::Cast { + expr, data_type, .. + } => write!(f, "CAST({} AS {})", expr, data_type), Expr::TryCast { expr, data_type } => write!(f, "TRY_CAST({} AS {})", expr, data_type), Expr::Extract { field, expr } => write!(f, "EXTRACT({} FROM {})", field, expr), Expr::Collate { expr, collation } => write!(f, "{} COLLATE {}", expr, collation), diff --git a/src/parser.rs b/src/parser.rs index 3389ab8d1..0a1d5c44d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -740,9 +740,19 @@ impl<'a> Parser<'a> { self.expect_keyword(Keyword::AS)?; let data_type = self.parse_data_type()?; self.expect_token(&Token::RParen)?; + let collation = if self.parse_keywords(&[Keyword::COLLATE]) { + match self.next_token() { + Token::Word(w) => Some(w.value), + unexpected => self.expected("identifier", unexpected)?, + } + } else { + None + }; + Ok(Expr::Cast { expr: Box::new(expr), data_type, + collation, }) } @@ -1240,6 +1250,7 @@ impl<'a> Parser<'a> { Ok(Expr::Cast { expr: Box::new(expr), data_type: self.parse_data_type()?, + collation: None, }) } @@ -2430,12 +2441,19 @@ impl<'a> Parser<'a> { Ok(DataType::BigInt(optional_precision?)) } } - Keyword::VARCHAR => Ok(DataType::Varchar(self.parse_optional_precision()?)), + Keyword::VARCHAR => Ok(DataType::Varchar(self.parse_optional_precision()?, None)), Keyword::CHAR | Keyword::CHARACTER => { - if self.parse_keyword(Keyword::VARYING) { - Ok(DataType::Varchar(self.parse_optional_precision()?)) - } else { - Ok(DataType::Char(self.parse_optional_precision()?)) + let is_var = self.parse_keyword(Keyword::VARYING); + let precision = self.parse_optional_precision()?; + let mut charset: Option = None; + // handle CAST('test' AS CHAR CHARACTER SET utf8mb4); + if self.parse_keywords(&[Keyword::CHARACTER, Keyword::SET]) { + let name = self.parse_identifier()?; + charset = Some(name.value); + } + match is_var { + true => Ok(DataType::Varchar(precision, charset)), + _ => Ok(DataType::Char(precision, charset)), } } Keyword::UUID => Ok(DataType::Uuid), From e745dc167f14a86f80411ec66bf5be8b88289aee Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 30 Mar 2022 15:37:29 +0800 Subject: [PATCH 04/12] fix: fix compile error when running test cases --- src/ast/data_type.rs | 4 ++-- src/parser.rs | 4 ++-- tests/sqlparser_common.rs | 6 ++++-- tests/sqlparser_postgres.rs | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index 602d17dd8..4bdfc067b 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -28,7 +28,7 @@ pub enum DataType { /// Fixed-length character type e.g. CHAR(10), CHAR CHARACTER set utf8mb4 Char(Option, Option), /// Variable-length character type e.g. VARCHAR(10) - Varchar(Option, Option), + Varchar(Option), /// Uuid type Uuid, /// Large character object e.g. CLOB(1000) @@ -95,7 +95,7 @@ impl fmt::Display for DataType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { DataType::Char(size, _) => format_type_with_optional_length(f, "CHAR", size, false), - DataType::Varchar(size, _) => { + DataType::Varchar(size) => { format_type_with_optional_length(f, "CHARACTER VARYING", size, false) } DataType::Uuid => write!(f, "UUID"), diff --git a/src/parser.rs b/src/parser.rs index 0a1d5c44d..5049af04f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2441,7 +2441,7 @@ impl<'a> Parser<'a> { Ok(DataType::BigInt(optional_precision?)) } } - Keyword::VARCHAR => Ok(DataType::Varchar(self.parse_optional_precision()?, None)), + Keyword::VARCHAR => Ok(DataType::Varchar(self.parse_optional_precision()?)), Keyword::CHAR | Keyword::CHARACTER => { let is_var = self.parse_keyword(Keyword::VARYING); let precision = self.parse_optional_precision()?; @@ -2452,7 +2452,7 @@ impl<'a> Parser<'a> { charset = Some(name.value); } match is_var { - true => Ok(DataType::Varchar(precision, charset)), + true => Ok(DataType::Varchar(precision)), _ => Ok(DataType::Char(precision, charset)), } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 16763b008..7202be240 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1307,7 +1307,8 @@ fn parse_cast() { assert_eq!( &Expr::Cast { expr: Box::new(Expr::Identifier(Ident::new("id"))), - data_type: DataType::BigInt(None) + data_type: DataType::BigInt(None), + collation: None }, expr_from_projection(only(&select.projection)) ); @@ -1317,7 +1318,8 @@ fn parse_cast() { assert_eq!( &Expr::Cast { expr: Box::new(Expr::Identifier(Ident::new("id"))), - data_type: DataType::TinyInt(None) + data_type: DataType::TinyInt(None), + collation: None }, expr_from_projection(only(&select.projection)) ); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index b0bda1438..50cdeb9a0 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -837,7 +837,8 @@ fn parse_array_index_expr() { })), data_type: DataType::Array(Box::new(DataType::Array(Box::new(DataType::Int( None - ))))) + ))))), + collation: None, }))), indexs: vec![num[1].clone(), num[2].clone()], }, From 79073bdc8cebcaa637806a9826fd90df960cd503 Mon Sep 17 00:00:00 2001 From: imotai Date: Sat, 2 Apr 2022 14:35:26 +0800 Subject: [PATCH 05/12] feat: support double at operator --- examples/transacion_variables.sql | 1 + src/ast/operator.rs | 3 +++ src/parser.rs | 10 ++++++++++ src/tokenizer.rs | 11 ++++++++++- 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 examples/transacion_variables.sql diff --git a/examples/transacion_variables.sql b/examples/transacion_variables.sql new file mode 100644 index 000000000..ba8ca971e --- /dev/null +++ b/examples/transacion_variables.sql @@ -0,0 +1 @@ +select @@session.transaction_read_only diff --git a/src/ast/operator.rs b/src/ast/operator.rs index 244ea517c..2a63d2b49 100644 --- a/src/ast/operator.rs +++ b/src/ast/operator.rs @@ -34,6 +34,8 @@ pub enum UnaryOperator { PGPrefixFactorial, /// Absolute value, e.g. `@ -9` (PostgreSQL-specific) PGAbs, + /// operators for mysql system variables + DoubleAt, } impl fmt::Display for UnaryOperator { @@ -48,6 +50,7 @@ impl fmt::Display for UnaryOperator { UnaryOperator::PGPostfixFactorial => "!", UnaryOperator::PGPrefixFactorial => "!!", UnaryOperator::PGAbs => "@", + UnaryOperator::DoubleAt => "@@", }) } } diff --git a/src/parser.rs b/src/parser.rs index 6dbf5d407..d3f8d265f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -474,6 +474,16 @@ impl<'a> Parser<'a> { expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?), }) } + tok @ Token::DoubleAt if dialect_of!(self is MySqlDialect) => { + let op = match tok { + Token::DoubleAt => UnaryOperator::DoubleAt, + _ => unreachable!(), + }; + Ok(Expr::UnaryOp { + op, + expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?), + }) + } tok @ Token::DoubleExclamationMark | tok @ Token::PGSquareRoot | tok @ Token::PGCubeRoot diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 4fb962555..f321bbd68 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -133,6 +133,8 @@ pub enum Token { ExclamationMark, /// Double Exclamation Mark `!!` used for PostgreSQL prefix factorial operator DoubleExclamationMark, + /// Double `@@` used for mysql system variables + DoubleAt, /// AtSign `@` used for PostgreSQL abs operator AtSign, /// `|/`, a square root math operator in PostgreSQL @@ -187,6 +189,7 @@ impl fmt::Display for Token { Token::Sharp => f.write_str("#"), Token::ExclamationMark => f.write_str("!"), Token::DoubleExclamationMark => f.write_str("!!"), + Token::DoubleAt => f.write_str("@@"), Token::Tilde => f.write_str("~"), Token::TildeAsterisk => f.write_str("~*"), Token::ExclamationMarkTilde => f.write_str("!~"), @@ -601,7 +604,13 @@ impl<'a> Tokenizer<'a> { } } '#' => self.consume_and_return(chars, Token::Sharp), - '@' => self.consume_and_return(chars, Token::AtSign), + '@' => { + chars.next(); // consume + match chars.peek() { + Some('@') => self.consume_and_return(chars, Token::DoubleAt), + _ => Ok(Some(Token::AtSign)), + } + } '?' => self.consume_and_return(chars, Token::Placeholder(String::from("?"))), '$' => { chars.next(); From fcc72da03543381f39342bdee63ebcc1490618a1 Mon Sep 17 00:00:00 2001 From: imotai Date: Sat, 2 Apr 2022 14:43:39 +0800 Subject: [PATCH 06/12] feat: add cast sql example --- examples/cast.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 examples/cast.sql diff --git a/examples/cast.sql b/examples/cast.sql new file mode 100644 index 000000000..67baab05e --- /dev/null +++ b/examples/cast.sql @@ -0,0 +1,2 @@ +SELECT CAST('test' AS CHAR CHARACTER SET utf8mb4 COLLATE utf8mb4_bin); + From a5f078c11944f61cac032c2e56a87185fd3f64a8 Mon Sep 17 00:00:00 2001 From: imotai Date: Sat, 2 Apr 2022 14:47:21 +0800 Subject: [PATCH 07/12] feat: update cast.sql in examples --- examples/cast.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cast.sql b/examples/cast.sql index 67baab05e..7eb78e0c5 100644 --- a/examples/cast.sql +++ b/examples/cast.sql @@ -1,2 +1,2 @@ -SELECT CAST('test' AS CHAR CHARACTER SET utf8mb4 COLLATE utf8mb4_bin); +SELECT CAST('test' AS CHAR CHARACTER SET utf8mb4) COLLATE utf8mb4_bin; From cb4c67854aa05408e8763c8fc978861ba2bb7d61 Mon Sep 17 00:00:00 2001 From: imotai Date: Mon, 11 Apr 2022 18:07:13 +0800 Subject: [PATCH 08/12] fix: support parse single quota id --- examples/show_var.sql | 1 + src/parser.rs | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 examples/show_var.sql diff --git a/examples/show_var.sql b/examples/show_var.sql new file mode 100644 index 000000000..af0524019 --- /dev/null +++ b/examples/show_var.sql @@ -0,0 +1 @@ + show session variables where variable_name in ('max_allowed_packet','system_time_zone','time_zone','auto_increment_increment') diff --git a/src/parser.rs b/src/parser.rs index fbc0ab614..423c5b421 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2638,6 +2638,9 @@ impl<'a> Parser<'a> { Token::Word(w) => { idents.push(w.to_ident()); } + Token::SingleQuotedString(s) => { + idents.push(Ident::with_quote('\'', s)); + } Token::EOF => break, _ => {} } From 22a6acfd5b3c10f213e78c56b1563ed6e7337950 Mon Sep 17 00:00:00 2001 From: imotai Date: Thu, 14 Apr 2022 17:58:40 +0800 Subject: [PATCH 09/12] feat: support show tables --- examples/show_full.sql | 1 + src/ast/mod.rs | 27 +++++++++++++++++++++++++++ src/ast/test.yml | 32 ++++++++++++++++++++++++++++++++ src/parser.rs | 41 ++++++++++++++++++++++++++++------------- 4 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 examples/show_full.sql create mode 100644 src/ast/test.yml diff --git a/examples/show_full.sql b/examples/show_full.sql new file mode 100644 index 000000000..9965f65df --- /dev/null +++ b/examples/show_full.sql @@ -0,0 +1 @@ +show full tables from `host_monitor_db`; diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 90ec72693..a0aefdc19 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -864,6 +864,15 @@ pub enum Statement { obj_type: ShowCreateObject, obj_name: ObjectName, }, + /// SHOW Tables + /// + /// Note: this is a MySQL-specific statement. + ShowTables { + extended: bool, + full: bool, + db_name: ObjectName, + filter: Option, + }, /// SHOW COLUMNS /// /// Note: this is a MySQL-specific statement. @@ -1498,6 +1507,24 @@ impl fmt::Display for Statement { )?; Ok(()) } + Statement::ShowTables { + extended, + full, + db_name, + filter, + } => { + write!( + f, + "SHOW {extended}{full}TABLES FROM {db_name}", + extended = if *extended { "EXTENDED " } else { "" }, + full = if *full { "FULL " } else { "" }, + db_name = db_name, + )?; + if let Some(filter) = filter { + write!(f, " {}", filter)?; + } + Ok(()) + } Statement::ShowColumns { extended, full, diff --git a/src/ast/test.yml b/src/ast/test.yml new file mode 100644 index 000000000..e90061597 --- /dev/null +++ b/src/ast/test.yml @@ -0,0 +1,32 @@ +name: comment_trigger +jobs: + pr_trigger: + runs-on: ubuntu-latest + if: github.event.comment.body == 'build' && ${{ github.event.issue.pull_request }} + steps: + - name: Get PR SHA + id: sha + uses: actions/github-script@v4 + with: + result-encoding: string + script: | + const { owner, repo, number } = context.issue; + const pr = await github.pulls.get({ + owner, + repo, + pull_number: number, + }); + return pr.data.head.sha + - name: trigger_pr + uses: actions/github-script@v4 + with: + script: | + github.rest.actions.createWorkflowDispatch({ + owner, + repo, + workflow_id:'pr.yaml', + ref:${{ steps.sha.outputs.result }} + }); +on: + issue_comment: + types: [created, edited] \ No newline at end of file diff --git a/src/parser.rs b/src/parser.rs index 423c5b421..797513a9d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3134,19 +3134,34 @@ impl<'a> Parser<'a> { pub fn parse_show_columns(&mut self) -> Result { let extended = self.parse_keyword(Keyword::EXTENDED); let full = self.parse_keyword(Keyword::FULL); - self.expect_one_of_keywords(&[Keyword::COLUMNS, Keyword::FIELDS])?; - self.expect_one_of_keywords(&[Keyword::FROM, Keyword::IN])?; - let table_name = self.parse_object_name()?; - // MySQL also supports FROM here. In other words, MySQL - // allows both FROM FROM and FROM .
, - // while we only support the latter for now. - let filter = self.parse_show_statement_filter()?; - Ok(Statement::ShowColumns { - extended, - full, - table_name, - filter, - }) + if self.parse_keyword(Keyword::TABLES) { + self.expect_one_of_keywords(&[Keyword::FROM, Keyword::IN])?; + let db_name = self.parse_object_name()?; + // MySQL also supports FROM here. In other words, MySQL + // allows both FROM
FROM and FROM .
, + // while we only support the latter for now. + let filter = self.parse_show_statement_filter()?; + Ok(Statement::ShowTables { + extended, + full, + db_name, + filter, + }) + } else { + self.expect_one_of_keywords(&[Keyword::COLUMNS, Keyword::FIELDS])?; + self.expect_one_of_keywords(&[Keyword::FROM, Keyword::IN])?; + let table_name = self.parse_object_name()?; + // MySQL also supports FROM here. In other words, MySQL + // allows both FROM
FROM and FROM .
, + // while we only support the latter for now. + let filter = self.parse_show_statement_filter()?; + Ok(Statement::ShowColumns { + extended, + full, + table_name, + filter, + }) + } } pub fn parse_show_statement_filter( From 706d6b594a0836f7e48a8b994a5715d8410950cb Mon Sep 17 00:00:00 2001 From: imotai Date: Tue, 26 Apr 2022 16:37:39 +0800 Subject: [PATCH 10/12] fix: remove unused file --- src/ast/test.yml | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 src/ast/test.yml diff --git a/src/ast/test.yml b/src/ast/test.yml deleted file mode 100644 index e90061597..000000000 --- a/src/ast/test.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: comment_trigger -jobs: - pr_trigger: - runs-on: ubuntu-latest - if: github.event.comment.body == 'build' && ${{ github.event.issue.pull_request }} - steps: - - name: Get PR SHA - id: sha - uses: actions/github-script@v4 - with: - result-encoding: string - script: | - const { owner, repo, number } = context.issue; - const pr = await github.pulls.get({ - owner, - repo, - pull_number: number, - }); - return pr.data.head.sha - - name: trigger_pr - uses: actions/github-script@v4 - with: - script: | - github.rest.actions.createWorkflowDispatch({ - owner, - repo, - workflow_id:'pr.yaml', - ref:${{ steps.sha.outputs.result }} - }); -on: - issue_comment: - types: [created, edited] \ No newline at end of file From 4cc88318dd4f8457539c14ec6749dcd4e780b78e Mon Sep 17 00:00:00 2001 From: imotai Date: Thu, 28 Apr 2022 12:31:32 +0800 Subject: [PATCH 11/12] feat: merge from main --- src/tokenizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 16f9c8a46..71870dfe3 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -625,7 +625,7 @@ impl<'a> Tokenizer<'a> { _ => Ok(Some(Token::Tilde)), } } - '#' => { + '#' => { chars.next(); match chars.peek() { Some('>') => { From 462aa18828473a1ee6ee1340f39bd9f001bd4993 Mon Sep 17 00:00:00 2001 From: imotai Date: Thu, 28 Apr 2022 12:36:45 +0800 Subject: [PATCH 12/12] fix: fix error from clippy check --- tests/sqlparser_common.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0506d9686..e2e5168a2 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1384,7 +1384,8 @@ fn parse_cast() { assert_eq!( &Expr::Cast { expr: Box::new(Expr::Identifier(Ident::new("id"))), - data_type: DataType::Nvarchar(Some(50)) + data_type: DataType::Nvarchar(Some(50)), + collation: None, }, expr_from_projection(only(&select.projection)) );