Include comment text in token

Author: harupy

compiler/parser/python.lalrpop

@@ -1463,6 +1463,6 @@ extern {
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,
-        "#" => lexer::Tok::Comment,
+        "#" => lexer::Tok::Comment(_),
     }
 }

compiler/parser/src/lexer.rs

@@ -444,16 +444,17 @@ where
     /// Skip everything until end of line
     fn lex_comment(&mut self) -> LexResult {
         let start_pos = self.get_pos();
-        self.next_char();
+        let mut value = String::new();
+        value.push(self.next_char().unwrap());
         loop {
             match self.window[0] {
                 Some('\n') | None => {
                     let end_pos = self.get_pos();
-                    return Ok((start_pos, Tok::Comment, end_pos));
+                    return Ok((start_pos, Tok::Comment(value), end_pos));
                 }
                 Some(_) => {}
             }
-            self.next_char();
+            value.push(self.next_char().unwrap());
         }
     }
 
@@ -1266,7 +1267,7 @@ mod tests {
             fn $name() {
                 let source = format!(r"99232  # {}", $eol);
                 let tokens = lex_source(&source);
-                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment(format!("# {}", $eol)), Tok::Newline]);
             }
             )*
         }
@@ -1290,7 +1291,7 @@ mod tests {
                     tokens,
                     vec![
                         Tok::Int { value: BigInt::from(123) },
-                        Tok::Comment,
+                        Tok::Comment("# Foo".to_string()),
                         Tok::Newline,
                         Tok::Int { value: BigInt::from(456) },
                         Tok::Newline,

compiler/parser/src/parser.rs

@@ -96,7 +96,7 @@ pub fn parse_located(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
 
     python::TopParser::new()
         .parse(tokenizer)
@@ -112,7 +112,7 @@ pub fn parse_tokens(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
 
     python::TopParser::new()
         .parse(tokenizer)

compiler/parser/src/token.rs

@@ -37,7 +37,7 @@ pub enum Tok {
     Rsqb,
     Colon,
     Comma,
-    Comment,
+    Comment(String),
     Semi,
     Plus,
     Minus,
@@ -148,7 +148,7 @@ impl fmt::Display for Tok {
             Rsqb => f.write_str("']'"),
             Colon => f.write_str("':'"),
             Comma => f.write_str("','"),
-            Comment => f.write_str("#"),
+            Comment(value) => f.write_str(value),
             Semi => f.write_str("';'"),
             Plus => f.write_str("'+'"),
             Minus => f.write_str("'-'"),