Merge pull request RustPython#4359 from yt2b/check_bom

Add BOM check

Author: youknowone

Lib/test/test_importlib/source/test_source_encoding.py

@@ -76,14 +76,10 @@ def test_encoding_on_second_line(self):
         source = b"#/usr/bin/python\n" + self.create_source('Latin-1')
         self.run_test(source)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     # [BOM]
     def test_bom(self):
         self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8'))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     # [BOM and utf-8]
     def test_bom_and_utf_8(self):
         source = codecs.BOM_UTF8 + self.create_source('utf-8')

compiler/parser/src/lexer.rs

@@ -236,6 +236,10 @@ where
         lxr.window.slide();
         lxr.window.slide();
         lxr.window.slide();
+        // TODO: Handle possible mismatch between BOM and explicit encoding declaration.
+        if let Some('\u{feff}') = lxr.window[0] {
+            lxr.window.slide();
+        }
         // Start at top row (=1) left column (=1)
         lxr.location.reset();
         lxr