Merge pull request RustPython#1614 from yanganto/update-sys-module

Update sys module when stdlib_inits is changed

Author: windelbouwman

src/main.rs

@@ -7,12 +7,12 @@ extern crate log;
 use clap::{App, AppSettings, Arg, ArgMatches};
 use rustpython_compiler::compile;
 use rustpython_vm::{
-    import, match_class,
+    match_class,
     obj::{objint::PyInt, objtuple::PyTuple, objtype},
     print_exception,
     pyobject::{ItemProtocol, PyResult},
     scope::Scope,
-    util, PySettings, VirtualMachine,
+    util, InitParameter, PySettings, VirtualMachine,
 };
 
 use std::convert::TryInto;
@@ -29,7 +29,13 @@ fn main() {
     env_logger::init();
     let app = App::new("RustPython");
     let matches = parse_arguments(app);
-    let settings = create_settings(&matches);
+    let mut settings = create_settings(&matches);
+
+    // We only include the standard library bytecode in WASI when initializing
+    if cfg!(target_os = "wasi") {
+        settings.initialization_parameter = InitParameter::InitializeInternal;
+    }
+
     let vm = VirtualMachine::new(settings);
 
     let res = run_rustpython(&vm, &matches);
@@ -324,9 +330,6 @@ fn write_profile(matches: &ArgMatches) -> Result<(), Box<dyn std::error::Error>>
 }
 
 fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
-    // We only include the standard library bytecode in WASI
-    import::init_importlib(&vm, cfg!(not(target_os = "wasi")))?;
-
     if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
         let sys_path = vm.get_attribute(vm.sys_module.clone(), "path")?;
         for (i, path) in std::env::split_paths(paths).enumerate() {

vm/src/import.rs

@@ -9,31 +9,38 @@ use crate::obj::{objcode, objtype};
 use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue};
 use crate::scope::Scope;
 use crate::version::get_git_revision;
-use crate::vm::VirtualMachine;
+use crate::vm::{InitParameter, VirtualMachine};
 #[cfg(feature = "rustpython-compiler")]
 use rustpython_compiler::compile;
 
-pub fn init_importlib(vm: &VirtualMachine, external: bool) -> PyResult {
+pub fn init_importlib(vm: &VirtualMachine, initialize_parameter: InitParameter) -> PyResult {
     flame_guard!("init importlib");
     let importlib = import_frozen(vm, "_frozen_importlib")?;
     let impmod = import_builtin(vm, "_imp")?;
     let install = vm.get_attribute(importlib.clone(), "_install")?;
     vm.invoke(&install, vec![vm.sys_module.clone(), impmod])?;
     vm.import_func
         .replace(vm.get_attribute(importlib.clone(), "__import__")?);
-    if external && cfg!(feature = "rustpython-compiler") {
-        flame_guard!("install_external");
-        let install_external =
-            vm.get_attribute(importlib.clone(), "_install_external_importers")?;
-        vm.invoke(&install_external, vec![])?;
-        // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
-        let importlib_external = vm.import("_frozen_importlib_external", &[], 0)?;
-        let mut magic = get_git_revision().into_bytes();
-        magic.truncate(4);
-        if magic.len() != 4 {
-            magic = rand::thread_rng().gen::<[u8; 4]>().to_vec();
+
+    match initialize_parameter {
+        InitParameter::InitializeExternal if cfg!(feature = "rustpython-compiler") => {
+            flame_guard!("install_external");
+            let install_external =
+                vm.get_attribute(importlib.clone(), "_install_external_importers")?;
+            vm.invoke(&install_external, vec![])?;
+            // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
+            let importlib_external = vm.import("_frozen_importlib_external", &[], 0)?;
+            let mut magic = get_git_revision().into_bytes();
+            magic.truncate(4);
+            if magic.len() != 4 {
+                magic = rand::thread_rng().gen::<[u8; 4]>().to_vec();
+            }
+            vm.set_attr(&importlib_external, "MAGIC_NUMBER", vm.ctx.new_bytes(magic))?;
+        }
+        InitParameter::NoInitialize => {
+            panic!("Import library initialize should be InitializeInternal or InitializeExternal");
         }
-        vm.set_attr(&importlib_external, "MAGIC_NUMBER", vm.ctx.new_bytes(magic))?;
+        _ => {}
     }
     Ok(vm.get_none())
 }

vm/src/lib.rs

@@ -73,7 +73,7 @@ mod vm;
 
 // pub use self::pyobject::Executor;
 pub use self::exceptions::{print_exception, write_exception};
-pub use self::vm::{PySettings, VirtualMachine};
+pub use self::vm::{InitParameter, PySettings, VirtualMachine};
 pub use rustpython_bytecode::*;
 
 #[doc(hidden)]

vm/src/vm.rs

@@ -69,10 +69,18 @@ pub struct VirtualMachine {
     pub settings: PySettings,
     pub recursion_limit: Cell<usize>,
     pub codec_registry: RefCell<Vec<PyObjectRef>>,
+    pub initialized: bool,
 }
 
 pub const NSIG: usize = 64;
 
+#[derive(Copy, Clone)]
+pub enum InitParameter {
+    NoInitialize,
+    InitializeInternal,
+    InitializeExternal,
+}
+
 /// Struct containing all kind of settings for the python vm.
 pub struct PySettings {
     /// -d command line switch
@@ -107,6 +115,10 @@ pub struct PySettings {
 
     /// sys.argv
     pub argv: Vec<String>,
+
+    /// Initialization parameter to decide to initialize or not,
+    /// and to decide the importer required external filesystem access or not
+    pub initialization_parameter: InitParameter,
 }
 
 /// Trace events for sys.settrace and sys.setprofile.
@@ -140,14 +152,15 @@ impl Default for PySettings {
             dont_write_bytecode: false,
             path_list: vec![],
             argv: vec![],
+            initialization_parameter: InitParameter::InitializeExternal,
         }
     }
 }
 
 impl VirtualMachine {
     /// Create a new `VirtualMachine` structure.
     pub fn new(settings: PySettings) -> VirtualMachine {
-        flame_guard!("init VirtualMachine");
+        flame_guard!("new VirtualMachine");
         let ctx = PyContext::new();
 
         // make a new module without access to the vm; doesn't
@@ -167,8 +180,9 @@ impl VirtualMachine {
         let profile_func = RefCell::new(ctx.none());
         let trace_func = RefCell::new(ctx.none());
         let signal_handlers = RefCell::new(arr![ctx.none(); 64]);
+        let initialize_parameter = settings.initialization_parameter;
 
-        let vm = VirtualMachine {
+        let mut vm = VirtualMachine {
             builtins: builtins.clone(),
             sys_module: sysmod.clone(),
             stdlib_inits,
@@ -185,6 +199,7 @@ impl VirtualMachine {
             settings,
             recursion_limit: Cell::new(512),
             codec_registry: RefCell::default(),
+            initialized: false,
         };
 
         objmodule::init_module_dict(
@@ -199,14 +214,32 @@ impl VirtualMachine {
             vm.new_str("sys".to_owned()),
             vm.get_none(),
         );
+        vm.initialize(initialize_parameter);
+        vm
+    }
 
-        builtins::make_module(&vm, builtins.clone());
-        sysmodule::make_module(&vm, sysmod, builtins);
+    pub fn initialize(&mut self, initialize_parameter: InitParameter) {
+        flame_guard!("init VirtualMachine");
 
-        #[cfg(not(target_arch = "wasm32"))]
-        import::import_builtin(&vm, "signal").expect("Couldn't initialize signal module");
+        match initialize_parameter {
+            InitParameter::NoInitialize => {}
+            _ => {
+                if self.initialized {
+                    panic!("Double Initialize Error");
+                }
 
-        vm
+                builtins::make_module(self, self.builtins.clone());
+                sysmodule::make_module(self, self.sys_module.clone(), self.builtins.clone());
+
+                #[cfg(not(target_arch = "wasm32"))]
+                import::import_builtin(self, "signal").expect("Couldn't initialize signal module");
+
+                import::init_importlib(self, initialize_parameter)
+                    .expect("Initialize importlib fail");
+
+                self.initialized = true;
+            }
+        }
     }
 
     pub fn run_code_obj(&self, code: PyCodeRef, scope: Scope) -> PyResult {

wasm/lib/src/vm_class.rs

@@ -7,10 +7,9 @@ use wasm_bindgen::prelude::*;
 
 use rustpython_compiler::compile;
 use rustpython_vm::function::PyFuncArgs;
-use rustpython_vm::import;
 use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue};
 use rustpython_vm::scope::{NameProtocol, Scope};
-use rustpython_vm::VirtualMachine;
+use rustpython_vm::{InitParameter, PySettings, VirtualMachine};
 
 use crate::browser_module::setup_browser_module;
 use crate::convert;
@@ -27,7 +26,13 @@ pub(crate) struct StoredVirtualMachine {
 
 impl StoredVirtualMachine {
     fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine {
-        let mut vm: VirtualMachine = Default::default();
+        let mut settings = PySettings::default();
+
+        // After js, browser modules injected, the VM will not be initialized.
+        settings.initialization_parameter = InitParameter::NoInitialize;
+
+        let mut vm: VirtualMachine = VirtualMachine::new(settings);
+
         vm.wasm_id = Some(id);
         let scope = vm.new_scope_with_builtins();
 
@@ -44,7 +49,7 @@ impl StoredVirtualMachine {
             setup_browser_module(&vm);
         }
 
-        import::init_importlib(&vm, false).unwrap();
+        vm.initialize(InitParameter::InitializeInternal);
 
         StoredVirtualMachine {
             vm,