Merge pull request RustPython#4229 from charliermarsh/charlie/tokens

Expose a method to parse AST from tokens directly

Author: youknowone

compiler/parser/src/parser.rs

@@ -5,6 +5,7 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
+use crate::lexer::LexResult;
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
 use std::iter;
@@ -80,6 +81,20 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
+// Parse a given token iterator.
+pub fn parse_tokens(
+    lxr: impl IntoIterator<Item = LexResult>,
+    mode: Mode,
+    source_path: &str,
+) -> Result<ast::Mod, ParseError> {
+    let marker_token = (Default::default(), mode.to_marker(), Default::default());
+    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+
+    python::TopParser::new()
+        .parse(tokenizer)
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;