Move pylib dependency out of vm

Author: youknowone

Cargo.lock

@@ -22,9 +22,9 @@ members = [
 default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
-stdlib = ["rustpython-stdlib"]
+stdlib = ["rustpython-stdlib", "rustpython-pylib"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
 zlib = ["stdlib", "rustpython-stdlib/zlib"]
@@ -35,7 +35,8 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 [dependencies]
 rustpython-compiler = { path = "compiler", version = "0.1.1" }
 rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
-rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
+rustpython-pylib = { path = "pylib", optional = true, default-features = false }
+rustpython-stdlib = { path = "stdlib", optional = true, default-features = false }
 rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
 
 cfg-if = "1.0.0"

Cargo.toml

@@ -0,0 +1 @@
+../Lib

pylib/Lib

@@ -6,27 +6,10 @@
 // so build.rs sets this env var
 pub const LIB_PATH: &str = match option_env!("win_lib_path") {
     Some(s) => s,
-    None => concat!(env!("CARGO_MANIFEST_DIR"), "/../../Lib"),
+    None => concat!(env!("CARGO_MANIFEST_DIR"), "/../Lib"),
 };
 
-use rustpython_compiler_core::FrozenModule;
-
-pub fn frozen_builtins() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(
-        dir = "../vm/Lib/python_builtins",
-        crate_name = "rustpython_compiler_core"
-    )
-}
-
-#[cfg(not(feature = "freeze-stdlib"))]
-pub fn frozen_core() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(
-        dir = "../vm/Lib/core_modules",
-        crate_name = "rustpython_compiler_core"
-    )
-}
-
 #[cfg(feature = "freeze-stdlib")]
-pub fn frozen_stdlib() -> impl Iterator<Item = (String, FrozenModule)> {
+pub fn frozen_stdlib() -> impl Iterator<Item = (String, rustpython_compiler_core::FrozenModule)> {
     rustpython_derive::py_freeze!(dir = "../Lib", crate_name = "rustpython_compiler_core")
 }

pylib/src/lib.rs

@@ -248,6 +248,38 @@ fn add_stdlib(vm: &mut VirtualMachine) {
     let _ = vm;
     #[cfg(feature = "stdlib")]
     vm.add_native_modules(rustpython_stdlib::get_module_inits());
+
+    // if we're on freeze-stdlib, the core stdlib modules will be included anyway
+    #[cfg(feature = "freeze-stdlib")]
+    vm.add_frozen(rustpython_pylib::frozen_stdlib());
+
+    #[cfg(not(feature = "freeze-stdlib"))]
+    {
+        use rustpython_vm::common::rc::PyRc;
+        let state = PyRc::get_mut(&mut vm.state).unwrap();
+
+        #[allow(clippy::needless_collect)] // false positive
+        let path_list: Vec<_> = state.settings.path_list.drain(..).collect();
+
+        // add the current directory to sys.path
+        state.settings.path_list.push("".to_owned());
+
+        // BUILDTIME_RUSTPYTHONPATH should be set when distributing
+        if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
+            state
+                .settings
+                .path_list
+                .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()))
+        } else {
+            #[cfg(feature = "rustpython-pylib")]
+            state
+                .settings
+                .path_list
+                .push(rustpython_pylib::LIB_PATH.to_owned())
+        }
+
+        state.settings.path_list.extend(path_list.into_iter());
+    }
 }
 
 /// Create settings by examining command line arguments and environment
@@ -264,21 +296,6 @@ fn create_settings(matches: &ArgMatches) -> Settings {
 
     let ignore_environment = settings.ignore_environment || settings.isolated;
 
-    // when rustpython-vm/pylib is enabled, Settings::default().path_list has pylib::LIB_PATH
-    let maybe_pylib = settings.path_list.pop();
-
-    // add the current directory to sys.path
-    settings.path_list.push("".to_owned());
-
-    // BUILDTIME_RUSTPYTHONPATH should be set when distributing
-    if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
-        settings
-            .path_list
-            .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()))
-    } else {
-        settings.path_list.extend(maybe_pylib);
-    }
-
     if !ignore_environment {
         settings.path_list.extend(get_paths("RUSTPYTHONPATH"));
         settings.path_list.extend(get_paths("PYTHONPATH"));

src/lib.rs

@@ -14,7 +14,7 @@ importlib = []
 encodings = ["importlib"]
 vm-tracing-logging = []
 flame-it = ["flame", "flamer"]
-freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
+freeze-stdlib = []
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
 compiler = ["parser", "codegen", "rustpython-compiler"]
@@ -31,7 +31,6 @@ rustpython-compiler-core = { path = "../compiler/core", version = "0.1.2" }
 rustpython-common = { path = "../common" }
 rustpython-derive = { path = "../derive", version = "0.1.2" }
 rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
-rustpython-pylib = { path = "../pylib", version = "0.1.0" }
 
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-bigint = { version = "0.4.3", features = ["serde"] }

vm/Cargo.toml

@@ -1,6 +1,6 @@
 use crate::bytecode::FrozenModule;
 
-pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
+pub fn core_frozen_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     let iter = std::iter::empty();
     macro_rules! ext_modules {
         ($iter:ident, ($modules:expr)) => {
@@ -20,16 +20,24 @@ pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     // Python modules that the vm calls into, but are not actually part of the stdlib. They could
     // in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
     // Includes _importlib_bootstrap and _importlib_bootstrap_external
-    ext_modules!(iter, (rustpython_pylib::frozen_builtins()));
+    ext_modules!(
+        iter,
+        (rustpython_derive::py_freeze!(
+            dir = "./Lib/python_builtins",
+            crate_name = "rustpython_compiler_core"
+        ))
+    );
 
-    #[cfg(not(feature = "freeze-stdlib"))]
     // core stdlib Python modules that the vm calls into, but are still used in Python
     // application code, e.g. copyreg
-    ext_modules!(iter, (rustpython_pylib::frozen_core()));
-
-    // if we're on freeze-stdlib, the core stdlib modules will be included anyway
-    #[cfg(feature = "freeze-stdlib")]
-    ext_modules!(iter, (rustpython_pylib::frozen_stdlib()));
+    #[cfg(not(feature = "freeze-stdlib"))]
+    ext_modules!(
+        iter,
+        (rustpython_derive::py_freeze!(
+            dir = "./Lib/core_modules",
+            crate_name = "rustpython_compiler_core"
+        ))
+    );
 
     iter
 }

vm/src/frozen.rs

@@ -176,7 +176,7 @@ impl VirtualMachine {
             panic!("Interpreters in same process must share the hash seed");
         }
 
-        let frozen = frozen::get_module_inits().collect();
+        let frozen = frozen::core_frozen_inits().collect();
         PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen;
 
         vm.builtins

vm/src/vm/mod.rs

@@ -89,7 +89,7 @@ impl Default for Settings {
             dev_mode: false,
             warn_default_encoding: false,
             warnopts: vec![],
-            path_list: vec![rustpython_pylib::LIB_PATH.to_owned()],
+            path_list: vec![],
             argv: vec![],
             hash_seed: None,
             stdio_unbuffered: false,

vm/src/vm/setting.rs

@@ -11,13 +11,15 @@ edition = "2021"
 crate-type = ["cdylib", "rlib"]
 
 [features]
-default = ["freeze-stdlib", "rustpython-stdlib"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+default = ["stdlib"]
+stdlib = ["freeze-stdlib", "rustpython-pylib", "rustpython-stdlib"]
+freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 no-start-func = []
 
 [dependencies]
 rustpython-common = { path = "../../common" }
 rustpython-parser = { path = "../../compiler/parser" }
+rustpython-pylib = { path = "../../pylib", default-features = false, optional = true }
 rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true }
 # make sure no threading! otherwise wasm build will fail
 rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] }

wasm/lib/Cargo.toml

@@ -42,6 +42,12 @@ impl StoredVirtualMachine {
         let mut settings = Settings::default();
         settings.allow_external_library = false;
         let interp = Interpreter::with_init(settings, |vm| {
+            #[cfg(feature = "stdlib")]
+            vm.add_native_modules(rustpython_stdlib::get_module_inits());
+
+            #[cfg(feature = "freeze-stdlib")]
+            vm.add_frozen(rustpython_pylib::frozen_stdlib());
+
             vm.wasm_id = Some(id);
 
             js_module::setup_js_module(vm);