Add a separate crate to package the python stdlib source code with

Author: coolreader18

Cargo.lock

@@ -8,7 +8,7 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace]
-members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze"]
+members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze", "vm/pylib-crate"]
 
 [[bench]]
 name = "bench"

Cargo.toml

@@ -157,10 +157,11 @@ struct PyCompileInput {
 }
 
 impl PyCompileInput {
-    fn compile(&self) -> Result<HashMap<String, FrozenModule>, Diagnostic> {
+    fn parse(&self) -> Result<PyCompileArgs, Diagnostic> {
         let mut module_name = None;
         let mut mode = None;
         let mut source: Option<CompilationSource> = None;
+        let mut crate_name = None;
 
         fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), Diagnostic> {
             if let Some(source) = source {
@@ -222,21 +223,29 @@ impl PyCompileInput {
                         kind: CompilationSourceKind::Dir(path),
                         span: extract_spans(&name_value).unwrap(),
                     });
+                } else if ident == "crate_name" {
+                    let name = match &name_value.lit {
+                        Lit::Str(s) => syn::Ident::new(&s.value(), s.span()),
+                        _ => bail_span!(name_value.lit, "source must be a string"),
+                    };
+                    crate_name = Some(name);
                 }
             }
         }
 
-        source
-            .ok_or_else(|| {
-                Diagnostic::span_error(
-                    self.span,
-                    "Must have either file or source in py_compile_bytecode!()",
-                )
-            })?
-            .compile(
-                mode.unwrap_or(compile::Mode::Exec),
-                module_name.unwrap_or_else(|| "frozen".to_owned()),
+        let source = source.ok_or_else(|| {
+            Diagnostic::span_error(
+                self.span,
+                "Must have either file or source in py_compile_bytecode!()",
             )
+        })?;
+
+        Ok(PyCompileArgs {
+            source,
+            mode: mode.unwrap_or(compile::Mode::Exec),
+            module_name: module_name.unwrap_or_else(|| "frozen".to_owned()),
+            crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(rustpython_vm)),
+        })
     }
 }
 
@@ -251,10 +260,19 @@ impl Parse for PyCompileInput {
     }
 }
 
+struct PyCompileArgs {
+    source: CompilationSource,
+    mode: compile::Mode,
+    module_name: String,
+    crate_name: syn::Ident,
+}
+
 pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
     let input: PyCompileInput = parse2(input)?;
+    let args = input.parse()?;
 
-    let code_map = input.compile()?;
+    let crate_name = args.crate_name;
+    let code_map = args.source.compile(args.mode, args.module_name)?;
 
     let modules = code_map
         .into_iter()
@@ -263,8 +281,8 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Dia
             let bytes = code.to_bytes();
             let bytes = LitByteStr::new(&bytes, Span::call_site());
             quote! {
-                #module_name.into() => ::rustpython_vm::bytecode::FrozenModule {
-                    code: ::rustpython_vm::bytecode::CodeObject::from_bytes(
+                #module_name.into() => ::#crate_name::bytecode::FrozenModule {
+                    code: ::#crate_name::bytecode::CodeObject::from_bytes(
                         #bytes
                     ).expect("Deserializing CodeObject failed"),
                     package: #package,
@@ -274,7 +292,7 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Dia
 
     let output = quote! {
         ({
-            use ::rustpython_vm::__exports::hashmap;
+            use ::#crate_name::__exports::hashmap;
             hashmap! {
                 #(#modules),*
             }

derive/src/compile_bytecode.rs

@@ -12,7 +12,7 @@ include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"]
 default = ["rustpython-parser", "rustpython-compiler"]
 vm-tracing-logging = []
 flame-it = ["flame", "flamer"]
-freeze-stdlib = []
+freeze-stdlib = ["rustpython-pylib"]
 
 ssl = ["openssl", "openssl-sys", "openssl-probe"]
 
@@ -37,10 +37,11 @@ rand_core = "0.5"
 getrandom = { version = "0.1", features = ["wasm-bindgen"] }
 mt19937 = "1.0"
 log = "0.4"
-rustpython-derive = {path = "../derive", version = "0.1.1"}
-rustpython-parser = {path = "../parser", optional = true, version = "0.1.1"}
-rustpython-compiler = {path = "../compiler", optional = true, version = "0.1.1"}
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1"}
+rustpython-derive = { path = "../derive", version = "0.1.2" }
+rustpython-parser = { path = "../parser", optional = true, version = "0.1.2" }
+rustpython-compiler = { path = "../compiler", optional = true, version = "0.1.2" }
+rustpython-bytecode = { path = "../bytecode", version = "0.1.2" }
+rustpython-pylib = { path = "pylib-crate", optional = true, version = "0.1.0" }
 serde = { version = "1.0.66", features = ["derive"] }
 byteorder = "1.2.6"
 regex = "1"

vm/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "rustpython-pylib"
+version = "0.1.0"
+authors = ["Noah <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+rustpython-derive = { version = "0.1.2", path = "../../derive" }
+rustpython-bytecode = { version = "0.1.2", path = "../../bytecode" }
+maplit = "1.0"

vm/pylib-crate/Cargo.toml

@@ -0,0 +1 @@
+../../Lib/

vm/pylib-crate/Lib

@@ -0,0 +1,28 @@
+//! This crate includes the compiled python bytecode of the RustPython standard library. The most
+//! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`,
+//! in order to automatically include the python part of the standard library into the binary.
+
+extern crate self as rustpython_pylib;
+
+use rustpython_bytecode::bytecode::{self, FrozenModule};
+use std::collections::HashMap;
+
+use rustpython_derive::py_compile_bytecode as _py_compile_bytecode;
+#[macro_export]
+macro_rules! py_compile_bytecode {
+    ($($arg:tt)*) => {{
+        #[macro_use]
+        mod __m {
+            $crate::_py_compile_bytecode!($($arg)*);
+        }
+        __proc_macro_call!()
+    }};
+}
+
+mod __exports {
+    pub use maplit::hashmap;
+}
+
+pub fn frozen_stdlib() -> HashMap<String, FrozenModule> {
+    py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib")
+}

vm/pylib-crate/src/lib.rs

@@ -29,7 +29,7 @@ pub fn get_module_inits() -> HashMap<String, FrozenModule> {
     // if we're on freeze-stdlib, the core stdlib modules will be included anyway
     #[cfg(feature = "freeze-stdlib")]
     {
-        ext_modules!(dir = "../Lib/");
+        modules.extend(rustpython_pylib::frozen_stdlib());
     }
 
     modules