Merge pull request RustPython#1092 from RustPython/bytecode-crate

Make parser and compiler optional features for vm crate.

Author: windelbouwman

Cargo.lock

@@ -16,6 +16,7 @@ path = "./benchmarks/bench.rs"
 log="0.4.1"
 env_logger="0.5.10"
 clap = "2.31.2"
+rustpython_compiler = {path = "compiler"}
 rustpython_parser = {path = "parser"}
 rustpython_vm = {path = "vm"}
 rustyline = "4.1.0"

Cargo.toml

@@ -8,18 +8,17 @@ extern crate rustpython_vm;
 extern crate rustyline;
 
 use clap::{App, Arg};
+use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
 use rustpython_parser::error::ParseError;
 use rustpython_vm::{
-    compile,
-    error::CompileError,
-    error::CompileErrorType,
     frame::Scope,
     import,
     obj::objstr,
     print_exception,
     pyobject::{ItemProtocol, PyResult},
     util, VirtualMachine,
 };
+
 use rustyline::{error::ReadlineError, Editor};
 use std::path::PathBuf;
 

src/main.rs

@@ -4,6 +4,9 @@ version = "0.1.0"
 authors = ["Shing Lyu <[email protected]>"]
 edition = "2018"
 
+[features]
+default = ["rustpython_parser", "rustpython_compiler"]
+
 [dependencies]
 # Crypto:
 digest = "0.8.1"
@@ -21,8 +24,8 @@ num-rational = "0.2.1"
 rand = "0.5"
 log = "0.3"
 rustpython_derive = {path = "../derive"}
-rustpython_parser = {path = "../parser"}
-rustpython_compiler = {path = "../compiler"}
+rustpython_parser = {path = "../parser", optional = true}
+rustpython_compiler = {path = "../compiler", optional = true}
 rustpython_bytecode = { path = "../bytecode" }
 serde = { version = "1.0.66", features = ["derive"] }
 serde_json = "1.0.26"

vm/Cargo.toml

@@ -10,7 +10,6 @@ use std::str;
 use num_bigint::Sign;
 use num_traits::{Signed, ToPrimitive, Zero};
 
-use crate::compile;
 use crate::obj::objbool;
 use crate::obj::objbytes::PyBytesRef;
 use crate::obj::objcode::PyCodeRef;
@@ -19,6 +18,8 @@ use crate::obj::objint::{self, PyIntRef};
 use crate::obj::objiter;
 use crate::obj::objstr::{self, PyString, PyStringRef};
 use crate::obj::objtype::{self, PyClassRef};
+#[cfg(feature = "rustpython_compiler")]
+use rustpython_compiler::compile;
 
 use crate::frame::Scope;
 use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs};
@@ -98,6 +99,7 @@ struct CompileArgs {
     optimize: OptionalArg<PyIntRef>,
 }
 
+#[cfg(feature = "rustpython_compiler")]
 fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult<PyCodeRef> {
     // TODO: compile::compile should probably get bytes
     let source = match args.source {
@@ -153,6 +155,7 @@ fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
 
 /// Implements `eval`.
 /// See also: https://docs.python.org/3/library/functions.html#eval
+#[cfg(feature = "rustpython_compiler")]
 fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
     // TODO: support any mapping for `locals`
     arg_check!(
@@ -184,6 +187,7 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
 
 /// Implements `exec`
 /// https://docs.python.org/3/library/functions.html#exec
+#[cfg(feature = "rustpython_compiler")]
 fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
     arg_check!(
         vm,
@@ -785,6 +789,15 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) {
     #[cfg(not(target_arch = "wasm32"))]
     let open = vm.ctx.new_rustfunc(io_open);
 
+    #[cfg(feature = "rustpython_compiler")]
+    {
+        extend_module!(vm, module, {
+            "compile" => ctx.new_rustfunc(builtin_compile),
+            "eval" => ctx.new_rustfunc(builtin_eval),
+            "exec" => ctx.new_rustfunc(builtin_exec),
+        });
+    }
+
     extend_module!(vm, module, {
         //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146
         "__name__" => ctx.new_str(String::from("__main__")),
@@ -799,15 +812,12 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) {
         "callable" => ctx.new_rustfunc(builtin_callable),
         "chr" => ctx.new_rustfunc(builtin_chr),
         "classmethod" => ctx.classmethod_type(),
-        "compile" => ctx.new_rustfunc(builtin_compile),
         "complex" => ctx.complex_type(),
         "delattr" => ctx.new_rustfunc(builtin_delattr),
         "dict" => ctx.dict_type(),
         "divmod" => ctx.new_rustfunc(builtin_divmod),
         "dir" => ctx.new_rustfunc(builtin_dir),
         "enumerate" => ctx.enumerate_type(),
-        "eval" => ctx.new_rustfunc(builtin_eval),
-        "exec" => ctx.new_rustfunc(builtin_exec),
         "float" => ctx.float_type(),
         "frozenset" => ctx.frozenset_type(),
         "filter" => ctx.filter_type(),

vm/src/builtins.rs

@@ -1,7 +1,7 @@
-use crate::compile;
 use crate::frame::Scope;
 use crate::pyobject::PyResult;
 use crate::vm::VirtualMachine;
+use rustpython_compiler::compile;
 
 pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult {
     match vm.compile(source, &compile::Mode::Eval, source_path.to_string()) {

vm/src/eval.rs

@@ -5,12 +5,13 @@
 use std::path::PathBuf;
 
 use crate::bytecode::CodeObject;
-use crate::compile;
 use crate::frame::Scope;
 use crate::obj::{objcode, objsequence, objstr};
 use crate::pyobject::{ItemProtocol, PyResult, PyValue};
 use crate::util;
 use crate::vm::VirtualMachine;
+#[cfg(feature = "rustpython_compiler")]
+use rustpython_compiler::compile;
 
 pub fn init_importlib(vm: &VirtualMachine) -> PyResult {
     let importlib = import_frozen(vm, "_frozen_importlib")?;
@@ -56,7 +57,7 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s
         import_frozen(vm, module_name)
     } else if vm.stdlib_inits.borrow().contains_key(module_name) {
         import_builtin(vm, module_name)
-    } else {
+    } else if cfg!(feature = "rustpython_compiler") {
         let notfound_error = &vm.ctx.exceptions.module_not_found_error;
         let import_error = &vm.ctx.exceptions.import_error;
 
@@ -72,9 +73,13 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s
             file_path.to_str().unwrap().to_string(),
             source,
         )
+    } else {
+        let notfound_error = &vm.ctx.exceptions.module_not_found_error;
+        Err(vm.new_exception(notfound_error.clone(), module_name.to_string()))
     }
 }
 
+#[cfg(feature = "rustpython_compiler")]
 pub fn import_file(
     vm: &VirtualMachine,
     module_name: &str,

vm/src/import.rs

@@ -44,6 +44,7 @@ pub mod macros;
 mod builtins;
 pub mod cformat;
 mod dictdatatype;
+#[cfg(feature = "rustpython_compiler")]
 pub mod eval;
 mod exceptions;
 pub mod format;
@@ -65,7 +66,6 @@ mod vm;
 pub use self::exceptions::print_exception;
 pub use self::vm::VirtualMachine;
 pub use rustpython_bytecode::*;
-pub use rustpython_compiler::*;
 
 #[doc(hidden)]
 pub mod __exports {

vm/src/lib.rs

@@ -1,10 +1,12 @@
+#[cfg(feature = "rustpython_parser")]
 mod ast;
 mod binascii;
 mod dis;
 mod hashlib;
 mod imp;
 mod itertools;
 mod json;
+#[cfg(feature = "rustpython_parser")]
 mod keyword;
 mod marshal;
 mod math;
@@ -16,6 +18,7 @@ pub mod socket;
 mod string;
 mod thread;
 mod time_module;
+#[cfg(feature = "rustpython_parser")]
 mod tokenize;
 mod warnings;
 mod weakref;
@@ -37,13 +40,11 @@ pub type StdlibInitFunc = Box<dyn Fn(&VirtualMachine) -> PyObjectRef>;
 pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
     #[allow(unused_mut)]
     let mut modules = hashmap! {
-        "ast".to_string() => Box::new(ast::make_module) as StdlibInitFunc,
-        "binascii".to_string() => Box::new(binascii::make_module),
-        "dis".to_string() => Box::new(dis::make_module),
+        "binascii".to_string() => Box::new(binascii::make_module) as StdlibInitFunc,
+        "dis".to_string() => Box::new(dis::make_module) as StdlibInitFunc,
         "hashlib".to_string() => Box::new(hashlib::make_module),
         "itertools".to_string() => Box::new(itertools::make_module),
         "json".to_string() => Box::new(json::make_module),
-        "keyword".to_string() => Box::new(keyword::make_module),
         "marshal".to_string() => Box::new(marshal::make_module),
         "math".to_string() => Box::new(math::make_module),
         "platform".to_string() => Box::new(platform::make_module),
@@ -53,12 +54,22 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
         "struct".to_string() => Box::new(pystruct::make_module),
         "_thread".to_string() => Box::new(thread::make_module),
         "time".to_string() => Box::new(time_module::make_module),
-        "tokenize".to_string() => Box::new(tokenize::make_module),
         "_weakref".to_string() => Box::new(weakref::make_module),
         "_imp".to_string() => Box::new(imp::make_module),
         "_warnings".to_string() => Box::new(warnings::make_module),
     };
 
+    // Insert parser related modules:
+    #[cfg(feature = "rustpython_parser")]
+    {
+        modules.insert(
+            "ast".to_string(),
+            Box::new(ast::make_module) as StdlibInitFunc,
+        );
+        modules.insert("keyword".to_string(), Box::new(keyword::make_module));
+        modules.insert("tokenize".to_string(), Box::new(tokenize::make_module));
+    }
+
     // disable some modules on WASM
     #[cfg(not(target_arch = "wasm32"))]
     {

vm/src/stdlib/mod.rs

@@ -12,8 +12,6 @@ use std::sync::{Mutex, MutexGuard};
 
 use crate::builtins;
 use crate::bytecode;
-use crate::compile;
-use crate::error::CompileError;
 use crate::frame::{ExecutionResult, Frame, FrameRef, Scope};
 use crate::frozen;
 use crate::function::PyFuncArgs;
@@ -38,6 +36,10 @@ use crate::pyobject::{
 use crate::stdlib;
 use crate::sysmodule;
 use num_bigint::BigInt;
+#[cfg(feature = "rustpython_compiler")]
+use rustpython_compiler::compile;
+#[cfg(feature = "rustpython_compiler")]
+use rustpython_compiler::error::CompileError;
 
 // use objects::objects;
 
@@ -249,6 +251,7 @@ impl VirtualMachine {
         self.new_exception(overflow_error, msg)
     }
 
+    #[cfg(feature = "rustpython_compiler")]
     pub fn new_syntax_error(&self, error: &CompileError) -> PyObjectRef {
         let syntax_error_type = self.ctx.exceptions.syntax_error.clone();
         let syntax_error = self.new_exception(syntax_error_type, error.to_string());
@@ -730,6 +733,7 @@ impl VirtualMachine {
         )
     }
 
+    #[cfg(feature = "rustpython_compiler")]
     pub fn compile(
         &self,
         source: &str,

vm/src/vm.rs

@@ -11,6 +11,7 @@ edition = "2018"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
+rustpython_compiler = { path = "../../compiler" }
 rustpython_parser = { path = "../../parser" }
 rustpython_vm = { path = "../../vm" }
 cfg-if = "0.1.2"

wasm/lib/Cargo.toml

@@ -5,7 +5,7 @@ use std::rc::{Rc, Weak};
 use js_sys::{Object, Reflect, SyntaxError, TypeError};
 use wasm_bindgen::prelude::*;
 
-use rustpython_vm::compile;
+use rustpython_compiler::{compile, error::CompileErrorType};
 use rustpython_vm::frame::{NameProtocol, Scope};
 use rustpython_vm::function::PyFuncArgs;
 use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue};
@@ -280,9 +280,7 @@ impl WASMVirtualMachine {
                 let code = vm.compile(&source, &mode, "<wasm>".to_string());
                 let code = code.map_err(|err| {
                     let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));
-                    if let rustpython_vm::error::CompileErrorType::Parse(ref parse_error) =
-                        err.error
-                    {
+                    if let CompileErrorType::Parse(ref parse_error) = err.error {
                         use rustpython_parser::error::ParseError;
                         if let ParseError::EOF(Some(ref loc))
                         | ParseError::ExtraToken((ref loc, ..))