diff --git a/compiler/parser/python.lalrpop b/compiler/parser/python.lalrpop index 9e6dd98832..90f4ce2582 100644 --- a/compiler/parser/python.lalrpop +++ b/compiler/parser/python.lalrpop @@ -1463,6 +1463,6 @@ extern { name => lexer::Tok::Name { name: }, "\n" => lexer::Tok::Newline, ";" => lexer::Tok::Semi, - "#" => lexer::Tok::Comment, + "#" => lexer::Tok::Comment(_), } } diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index ec4cfbed24..cb38503b3e 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -444,16 +444,17 @@ where /// Skip everything until end of line fn lex_comment(&mut self) -> LexResult { let start_pos = self.get_pos(); - self.next_char(); + let mut value = String::new(); + value.push(self.next_char().unwrap()); loop { match self.window[0] { Some('\n') | None => { let end_pos = self.get_pos(); - return Ok((start_pos, Tok::Comment, end_pos)); + return Ok((start_pos, Tok::Comment(value), end_pos)); } Some(_) => {} } - self.next_char(); + value.push(self.next_char().unwrap()); } } @@ -1266,7 +1267,7 @@ mod tests { fn $name() { let source = format!(r"99232 # {}", $eol); let tokens = lex_source(&source); - assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]); + assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment(format!("# {}", $eol)), Tok::Newline]); } )* } @@ -1290,7 +1291,7 @@ mod tests { tokens, vec![ Tok::Int { value: BigInt::from(123) }, - Tok::Comment, + Tok::Comment("# Foo".to_string()), Tok::Newline, Tok::Int { value: BigInt::from(456) }, Tok::Newline, diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs index aad0288284..4277d3002e 100644 --- a/compiler/parser/src/parser.rs +++ b/compiler/parser/src/parser.rs @@ -96,7 +96,7 @@ pub fn parse_located( let marker_token = (Default::default(), mode.to_marker(), Default::default()); let tokenizer = iter::once(Ok(marker_token)) .chain(lxr) - .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment)); + .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. })); python::TopParser::new() .parse(tokenizer) @@ -112,7 +112,7 @@ pub fn parse_tokens( let marker_token = (Default::default(), mode.to_marker(), Default::default()); let tokenizer = iter::once(Ok(marker_token)) .chain(lxr) - .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment)); + .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_))); python::TopParser::new() .parse(tokenizer) diff --git a/compiler/parser/src/token.rs b/compiler/parser/src/token.rs index c6185391db..72aa89d683 100644 --- a/compiler/parser/src/token.rs +++ b/compiler/parser/src/token.rs @@ -37,7 +37,7 @@ pub enum Tok { Rsqb, Colon, Comma, - Comment, + Comment(String), Semi, Plus, Minus, @@ -148,7 +148,7 @@ impl fmt::Display for Tok { Rsqb => f.write_str("']'"), Colon => f.write_str("':'"), Comma => f.write_str("','"), - Comment => f.write_str("#"), + Comment(value) => f.write_str(value), Semi => f.write_str("';'"), Plus => f.write_str("'+'"), Minus => f.write_str("'-'"),