Merge pull request RustPython#3014 from leesungbin/underscore_grammar

Solve some lexical error with underscore
sdww0 · Oct 18, 2021 · 7bcf2c3 · 7bcf2c3
2 parents dd3d289 + d7e9c7d
commit 7bcf2c3
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
@@ -179,8 +179,6 @@ def test_float_exponent_tokenization(self):
         self.assertEqual(1 if 0else 0, 0)
         self.assertRaises(SyntaxError, eval, "0 if 1Else 0")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_underscore_literals(self):
         for lit in VALID_UNDERSCORE_LITERALS:
             self.assertEqual(eval(lit), eval(lit.replace('_', '')))

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
@@ -316,10 +316,21 @@ where
 
             // 1e6 for example:
             if self.chr0 == Some('e') || self.chr0 == Some('E') {
+                if self.chr1 == Some('_') {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                        location: self.get_pos(),
+                    });
+                }
                 value_text.push(self.next_char().unwrap().to_ascii_lowercase());
-
                 // Optional +/-
                 if self.chr0 == Some('-') || self.chr0 == Some('+') {
+                    if self.chr1 == Some('_') {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                            location: self.get_pos(),
+                        });
+                    }
                     value_text.push(self.next_char().unwrap());
                 }