Remove xargo in favor of cargo's build-std feature (theseus-os#340)

* Inspired by theseus-os#337 but is a more complete solution, even though theseus-os#338 was already merged in. This solution is actually preferable and simplifies the build procedure and build dependencies.

* Removes all xargo-related files, e.g., `Xargo.toml` files.

* Since we no longer use xargo, which created its own sysroot directory in `$HOME/.xargo/`, we now manually create our own sysroot directory in `./build/deps/sysroot/` for future reproducible builds, e.g., with `libtheseus`.  The sysroot is now created by the `copy_latest_crate_objects` tool, and is an optional step.

Makefile

@@ -54,32 +54,6 @@ endif
 
 
 
-###################################################################################################
-### For ensuring that the host computer has the proper version of xargo
-###################################################################################################
-
-XARGO_CURRENT_SUPPORTED_VERSION := 0.3.22
-XARGO_OUTPUT=$(shell xargo --version 2>&1 | head -n 1 | grep -o 'xargo [0-9]*\.[0-9]*\.[0-9]*')
-
-check_xargo: 	
-ifneq (${BYPASS_XARGO_CHECK}, yes)
-ifneq (xargo ${XARGO_CURRENT_SUPPORTED_VERSION}, ${XARGO_OUTPUT})
-	@echo -e "\nError: your xargo version does not match our supported xargo version."
-	@echo -e "To install the proper version of xargo, run the following commands:\n"
-	@echo -e "   cargo uninstall xargo"
-	@echo -e '   rm -rf $$HOME/.xargo'   ## use single quotes to escape dollar sign
-	@echo -e "   rustup toolchain install stable"
-	@echo -e "   cargo +stable install --vers $(XARGO_CURRENT_SUPPORTED_VERSION) xargo"
-	@echo -e "   make clean\n"
-	@echo -e "Then you can retry building!\n"
-	@exit 1
-else
-	@echo -e '\nFound proper xargo version, proceeding with build...\n'
-endif ## RUSTC_CURRENT_SUPPORTED_VERSION != RUSTC_OUTPUT
-endif ## BYPASS_XARGO_CHECK
-
-
-
 ###################################################################################################
 ### This section contains targets to actually build Theseus components and create an iso file.
 ###################################################################################################
@@ -91,13 +65,14 @@ GRUB_ISOFILES := $(BUILD_DIR)/grub-isofiles
 OBJECT_FILES_BUILD_DIR := $(GRUB_ISOFILES)/modules
 DEBUG_SYMBOLS_DIR := $(BUILD_DIR)/debug_symbols
 DEPS_DIR := $(BUILD_DIR)/deps
-HOST_DEPS_DIR := $(BUILD_DIR)/deps/host_deps
+HOST_DEPS_DIR := $(DEPS_DIR)/host_deps
+DEPS_SYSROOT_DIR := $(DEPS_DIR)/sysroot
 THESEUS_BUILD_TOML := $(DEPS_DIR)/TheseusBuild.toml
 THESEUS_CARGO := $(ROOT_DIR)/tools/theseus_cargo
 THESEUS_CARGO_BIN := $(THESEUS_CARGO)/bin/theseus_cargo
 
 
-## This is the output path of the xargo command, defined by cargo (not our choice).
+## This is the default output path defined by cargo.
 nano_core_static_lib := $(ROOT_DIR)/target/$(TARGET)/$(BUILD_MODE)/libnano_core.a
 ## The directory where the nano_core source files are
 NANO_CORE_SRC_DIR := $(ROOT_DIR)/kernel/nano_core/src
@@ -134,7 +109,7 @@ APP_CRATE_NAMES += EXTRA_APP_CRATE_NAMES
 ### PHONY is the list of targets that *always* get rebuilt regardless of dependent files' modification timestamps.
 ### Most targets are PHONY because cargo itself handles whether or not to rebuild the Rust code base.
 .PHONY: all \
-		check_rustc check_xargo \
+		check_rustc \
 		clean run run_pause iso build cargo \
 		libtheseus \
 		simd_personality_sse build_sse simd_personality_avx build_avx \
@@ -189,23 +164,20 @@ build: $(nano_core_binary)
 ## Copy all object files into the main build directory and prepend the kernel or app prefix appropriately. 
 	@cargo run --release --manifest-path $(ROOT_DIR)/tools/copy_latest_crate_objects/Cargo.toml -- \
 		-i ./target/$(TARGET)/$(BUILD_MODE)/deps \
-		--sysroot "$(HOME)"/.xargo/lib/rustlib/$(TARGET)/lib/ \
 		--output-objects $(OBJECT_FILES_BUILD_DIR) \
 		--output-deps $(DEPS_DIR) \
+		--output-sysroot $(DEPS_SYSROOT_DIR)/lib/rustlib/$(TARGET)/lib \
 		-k ./kernel \
 		-a ./applications \
 		--kernel-prefix $(KERNEL_PREFIX) \
 		--app-prefix $(APP_PREFIX) \
 		-e "$(EXTRA_APP_CRATE_NAMES) libtheseus"
 
 ## Create the items needed for future out-of-tree builds that depend upon the parameters of this current build. 
-## This includes the target file, sysroot directory contents, host OS dependencies (proc macros, etc)., 
+## This includes the target file, host OS dependencies (proc macros, etc)., 
 ## and most importantly, a TOML file to describe these and other config variables.
 	@rm -rf $(THESEUS_BUILD_TOML)
 	@cp -vf $(CFG_DIR)/$(TARGET).json  $(DEPS_DIR)/
-	@cp -vf $(ROOT_DIR)/Xargo.toml  $(DEPS_DIR)/
-	@mkdir -p $(DEPS_DIR)/sysroot/lib/rustlib/$(TARGET)/lib/
-	@cp -r "$(HOME)"/.xargo/lib/rustlib/$(TARGET)  $(DEPS_DIR)/sysroot/lib/rustlib/
 	@mkdir -p $(HOST_DEPS_DIR)
 	@cp -f ./target/$(BUILD_MODE)/deps/*  $(HOST_DEPS_DIR)/
 	@echo -e 'target = "$(TARGET)"' >> $(THESEUS_BUILD_TOML)
@@ -245,23 +217,23 @@ endif
 
 
 ## This target invokes the actual Rust build process
-cargo: check_rustc check_xargo
+cargo: check_rustc 
 	@echo -e "\n=================== BUILDING ALL CRATES ==================="
 	@echo -e "\t TARGET: \"$(TARGET)\""
 	@echo -e "\t KERNEL_PREFIX: \"$(KERNEL_PREFIX)\""
 	@echo -e "\t APP_PREFIX: \"$(APP_PREFIX)\""
 	@echo -e "\t THESEUS_CONFIG (before build.rs script): \"$(THESEUS_CONFIG)\""
-	RUST_TARGET_PATH="$(CFG_DIR)" RUSTFLAGS="$(RUSTFLAGS)" xargo build  $(CARGOFLAGS)  $(RUST_FEATURES) --all --target $(TARGET)
+	RUST_TARGET_PATH="$(CFG_DIR)" RUSTFLAGS="$(RUSTFLAGS)" cargo build  $(CARGOFLAGS) $(BUILD_STD_CARGOFLAGS) $(RUST_FEATURES) --all --target $(TARGET)
 
-## We tried using the "xargo rustc" command here instead of "xargo build" to avoid xargo unnecessarily rebuilding core/alloc crates,
-## But it doesn't really seem to work (it's not the cause of xargo rebuilding everything).
-## For the "xargo rustc" command below, all of the arguments to cargo/xargo come before the "--",
+## We tried using the "cargo rustc" command here instead of "cargo build" to avoid cargo unnecessarily rebuilding core/alloc crates,
+## But it doesn't really seem to work (it's not the cause of cargo rebuilding everything).
+## For the "cargo rustc" command below, all of the arguments to cargo come before the "--",
 ## whereas all of the arguments to rustc come after the "--".
 # 	for kd in $(KERNEL_CRATE_NAMES) ; do  \
 # 		cd $${kd} ; \
 # 		echo -e "\n========= BUILDING KERNEL CRATE $${kd} ==========\n" ; \
 # 		RUST_TARGET_PATH="$(CFG_DIR)" RUSTFLAGS="$(RUSTFLAGS)" \
-# 			xargo rustc \
+# 			cargo rustc \
 # 			$(CARGOFLAGS) \
 # 			$(RUST_FEATURES) \
 # 			--target $(TARGET) ; \
@@ -270,7 +242,7 @@ cargo: check_rustc check_xargo
 # for app in $(APP_CRATE_NAMES) ; do  \
 # 	cd $${app} ; \
 # 	RUST_TARGET_PATH="$(CFG_DIR)" RUSTFLAGS="$(RUSTFLAGS)" \
-# 		xargo rustc \
+# 		cargo rustc \
 # 		$(CARGOFLAGS) \
 # 		--target $(TARGET) \
 # 		-- \

README.md

@@ -82,10 +82,6 @@ Note: building and running Theseus within a Docker container may be slower than
 Install the current Rust compiler and toolchain by following the [setup instructions here](https://www.rust-lang.org/en-US/install.html), which is just this command:    
 `curl https://sh.rustup.rs -sSf | sh`
 
-We also need to install Xargo, a drop-in replacement wrapper for Cargo that makes cross-compiling easier:    
-`rustup toolchain install stable`    
-`cargo +stable install --vers 0.3.22 xargo`    
-
 
 ## Building and Running
 To build and run Theseus in QEMU, simply run:   

applications/.old_static_binary_hello/Makefile

@@ -10,7 +10,7 @@ ifeq ($(BUILD_MODE), release)
 endif  ## otherwise, nothing, which means "debug" by default
 
 
-## this is the output path of the xargo command, not our choice.
+## this is the output path of the cargo command, not our choice.
 ## The name of the lib (lib_____.a) is defined by the target above.
 cargo-obj := target/$(TARGET)/$(BUILD_MODE)/hello
 
@@ -26,4 +26,4 @@ clean:
 
 
 build: 
-	RUST_TARGET_PATH="$(PWD)/../../cfg" RUSTFLAGS=$(RUSTFLAGS) xargo build $(CARGO_OPTIONS) --target $(TARGET)
+	RUST_TARGET_PATH="$(PWD)/../../cfg" RUSTFLAGS=$(RUSTFLAGS) cargo build $(CARGO_OPTIONS) --target $(TARGET)

book/src/build_process.md

@@ -3,7 +3,7 @@
 Theseus uses the [cargo virtual workspace](https://doc.rust-lang.org/cargo/reference/fest.html#the-workspace-section) feature to group all of the crates together into a single meta project, which significantly speeds up build times.
 
 The top-level Makefile basically just invokes the Rust toolchain and compiler, copies the built object files into a top-level build directory, and then generates a bootable .iso image using various bootloader tools.
-We build all of the Rust code using [`xargo`](https://github.com/ric/xargo), a cross-compiler toolchain that wraps the default Rust `cargo`.
+We build all of the Rust code using `cargo`, Rust's build tool and dependency manager.
 The only special action it takes is to build the `nano_core` separately and fully link it against the architecture-specific assembly code in `nano_core/boot` into a static binary.
 
 ## Debug vs. Release Mode

book/src/chapter_2.md

@@ -31,9 +31,6 @@ If you're on Mac OS, do the following:
 Install the current Rust compiler and toolchain by following the [setup instructions here](https://www.rust-lang.org/en-US/install.html), which is basically just this command:
 `curl https://sh.rustup.rs -sSf | sh`
 
-We also need to install Xargo, a drop-in replacement wrapper for Cargo that makes cross-compiling easier:
-`cargo install --vers 0.3.13 xargo`
-
 
 
 ### Building and Running

cfg/Config.mk

@@ -21,6 +21,7 @@ CFG_DIR := $(ROOT_DIR)/cfg
 KERNEL_PREFIX ?= k\#
 APP_PREFIX    ?= a\#
 
+
 ## Build modes: debug is development mode, release is with full optimizations.
 ## We build using release mode by default, because running in debug mode is prohibitively slow.
 ## You can set these on the command line like so: "make run BUILD_MODE=release"
@@ -31,6 +32,13 @@ ifeq ($(BUILD_MODE), release)
 	export override CARGOFLAGS += --release
 endif
 
+## Tell cargo to build our own target-specific version of the `core` and `alloc` crates.
+## Also ensure that core memory functions (e.g., memcpy) are included in the build and not name-mangled.
+## We keep these flags separate from the regular CARGOFLAGS for purposes of easily creating a sysroot directory.
+BUILD_STD_CARGOFLAGS += -Z unstable-options
+BUILD_STD_CARGOFLAGS += -Z build-std=core,alloc
+BUILD_STD_CARGOFLAGS += -Z build-std-features=compiler-builtins-mem
+
 
 ## emit obj gives us the object file for the crate, instead of an rlib that we have to unpack.
 RUSTFLAGS += --emit=obj

docker/Dockerfile

@@ -63,6 +63,3 @@ ENV PATH="$HOME/.cargo/bin:${PATH}"
 
 # Install the specific version of the Rust toolchain currently required by Theseus.
 RUN rustup toolchain install ${RUSTC_VERSION} && rustup component add rust-src
-
-# Install xargo (for cross compilation)
-RUN cargo install --vers 0.3.22 xargo

kernel/simd_personality/src/lib.rs

@@ -56,7 +56,7 @@ cfg_if! { if #[cfg(simd_personality)] {
 
 
 /* 
- * NOTE: now, we're using the compiler_builtins crate that is built by xargo by default, but we can switch back
+ * NOTE: now, we're using the compiler_builtins crate that is built by cargo's build-std feature by default, but we can switch back
  * to this one if needed since it does export different symbols based on Cargo.toml feature choices.
 // This crate is required for the SIMD environment,
 // so we can resolve symbols that the core lib requires. 

libtheseus/build.sh

@@ -4,7 +4,9 @@ set -e
 # capture all output to a file
 # script -e .script_output
 
-THESEUS_CARGO_PATH="../tools/theseus_cargo"
+LIBTHESEUS_DIR="$(dirname "$(readlink -f "$0")")"
+THESEUS_BASE_DIR=$LIBTHESEUS_DIR/..
+THESEUS_CARGO_PATH="$THESEUS_BASE_DIR/tools/theseus_cargo"
 
 export RUST_BACKTRACE=1
 
@@ -16,29 +18,39 @@ cargo install --force --path=$THESEUS_CARGO_PATH --root=$THESEUS_CARGO_PATH
 $THESEUS_CARGO_PATH/bin/theseus_cargo --input ../build/deps build
 
 
+
+## The newer NEWER raw cargo command that works without xargo at all, using cargo build-std.
+## We don't want to pass the build-std flags to cargo since that would re-build all of the core files from scratch. 
+## Instead, we want to re-use the existing pre-built core files from our previously-created sysroot output directory.
+# RUST_TARGET_PATH="$THESEUS_BASE_DIR/build/deps"   \
+# 	RUSTFLAGS="--emit=obj -C debuginfo=2 -C code-model=large -C relocation-model=static -D unused-must-use -Z merge-functions=disabled -Z share-generics=no --sysroot $THESEUS_BASE_DIR/build/deps/sysroot" \
+# 	cargo build --release  --verbose \
+# 	--target x86_64-theseus
+
+
 ## The "newer" raw cargo command that works without xargo at all. This is good because we can use a pre-built/distributed sysroot directory.
-# RUST_TARGET_PATH="/home/kevin/Dropbox/Theseus/build/deps"   \
-# 	RUSTFLAGS="--emit=obj -C debuginfo=2 -C code-model=large -C relocation-model=static -D unused-must-use -Z merge-functions=disabled -Z share-generics=no --sysroot /home/kevin/Dropbox/Theseus/build/deps/sysroot"  \
+# RUST_TARGET_PATH="$THESEUS_BASE_DIR/build/deps"   \
+# 	RUSTFLAGS="--emit=obj -C debuginfo=2 -C code-model=large -C relocation-model=static -D unused-must-use -Z merge-functions=disabled -Z share-generics=no --sysroot $THESEUS_BASE_DIR/build/deps/sysroot"  \
 # 	cargo build --release --verbose -vv \
 # 	--target x86_64-theseus
 
 
 ## The initial normal command that uses `xargo`
-# RUST_TARGET_PATH="/home/kevin/Dropbox/Theseus/cfg"  \
+# RUST_TARGET_PATH="$THESEUS_BASE_DIR/cfg"  \
 # 	RUSTFLAGS="--emit=obj -C debuginfo=2 -C code-model=large -C relocation-model=static -D unused-must-use -Z merge-functions=disabled -Z share-generics=no" \
 # 	xargo build  --release  --verbose -vv \
 # 	--target x86_64-theseus	
 
 
 
-# RUST_TARGET_PATH="/home/kevin/Dropbox/Theseus/cfg" \
+# RUST_TARGET_PATH="$THESEUS_BASE_DIR/cfg" \
 # 	rustc --crate-name libtheseus src/lib.rs  --crate-type lib \
 # 	--emit=dep-info,metadata,link \
 # 	-C opt-level=3 -C embed-bitcode=no -C codegen-units=1 -C metadata=43462c60d48a531a -C extra-filename=-43462c60d48a531a \
-# 	--out-dir /home/kevin/Dropbox/Theseus/libtheseus/target/x86_64-theseus/release/deps \
+# 	--out-dir $THESEUS_BASE_DIR/libtheseus/target/x86_64-theseus/release/deps \
 # 	--target x86_64-theseus \
-# 	-L dependency=/home/kevin/Dropbox/Theseus/target/x86_64-theseus/release/deps \
-# 	--extern rlibc=/home/kevin/Dropbox/Theseus/target/x86_64-theseus/release/deps/librlibc-4eb1a1ba9385f780.rmeta \
-# 	--extern serial_port=/home/kevin/Dropbox/Theseus/target/x86_64-theseus/release/deps/libserial_port-ce2d7a263b9ad06d.rmeta \
+# 	-L dependency=$THESEUS_BASE_DIR/target/x86_64-theseus/release/deps \
+# 	--extern rlibc=$THESEUS_BASE_DIR/target/x86_64-theseus/release/deps/librlibc-4eb1a1ba9385f780.rmeta \
+# 	--extern serial_port=$THESEUS_BASE_DIR/target/x86_64-theseus/release/deps/libserial_port-ce2d7a263b9ad06d.rmeta \
 # 	--emit=obj -C debuginfo=2 -C code-model=large -C relocation-model=static -D unused-must-use -Z merge-functions=disabled -Z share-generics=no \
 # 	--sysroot /home/kevin/.xargo

tools/copy_latest_crate_objects/src/main.rs

@@ -34,7 +34,6 @@ use std::{
         hash_map::Entry,
     },
     env,
-    ffi::OsStr,
     fs::{self, DirEntry, File},
     io::{self, BufRead},
     path::{Path, PathBuf},
@@ -65,15 +64,6 @@ fn main() -> Result<(), String> {
          typically the `target`, e.g., \"/path/to/Theseus/target/$TARGET/release/deps/\"", 
         "INPUT_DIR"
     );
-    opts.reqopt(
-        "",
-        "sysroot",
-        "(required) path to the target-specific subdirectory within the sysroot directory that holds fundamental Rust library files,
-         e.g., core, alloc, compiler_builtins. There should be at least an object file (.o) for each library. 
-         Typically the sysroot subdirectory is located at \"$HOME/.xargo/lib/rustlib/$TARGET/lib/\" if using xargo
-         for a cross-platform build (as with Theseus), otherwise you can locate the top-level sysroot using `rustc --print sysroot`.",
-        "SYSROOT_DIR"
-    );
     opts.reqopt(
         "", 
         "output-objects",  
@@ -88,6 +78,15 @@ fn main() -> Result<(), String> {
          typically part of the build directory, e.g., \"/path/to/build/deps/\"", 
         "OUTPUT_DIR"
     );
+    opts.optopt(
+        "", 
+        "output-sysroot",  
+        "path to the output directory where the sysroot files should be copied to,
+         which includes the .rmeta and .rlib files for fundamental Rust library crates, e.g., core, alloc, compiler_builtins. 
+         Typically this should be \"/path/to/build/deps/sysroot/lib/rustlib/$TARGET/lib/\".
+         If not provided, no sysroot output directory will be created.",
+        "OUTPUT_DIR"
+     );
     opts.reqopt(
         "k", 
         "kernel",  
@@ -152,7 +151,6 @@ fn main() -> Result<(), String> {
 
     // Parse the required command-line arguments.
     let input_dir          = matches.opt_str("i").expect("no -i or --input arg provided");
-    let sysroot_dir        = matches.opt_str("sysroot").expect("no --sysroot arg provided");
     let output_objects_dir = matches.opt_str("output-objects").expect("no --output-objects arg provided");
     let output_deps_dir    = matches.opt_str("output-deps").expect("no --output-deps arg provided");
     let kernel_arg         = matches.opt_str("k").expect("no -k or --kernel arg provided");
@@ -185,18 +183,6 @@ fn main() -> Result<(), String> {
     let extra_app_names = matches.opt_strs("e");
     app_crates_set.extend(extra_app_names.iter().flat_map(|n| n.split_whitespace()).map(|s| s.to_string()));
 
-    // Get all the object files from the provided sysroot directory. 
-    let object_file_extension = OsStr::new("o");
-    let core_object_file_paths = fs::read_dir(&sysroot_dir)
-        .expect("Could not access --sysroot directory")
-        .flat_map(|d| d)
-        .filter(|d| Path::new(&d.file_name()).extension() == Some(&object_file_extension))
-        .map(|d| d.path())
-        .collect::<Vec<PathBuf>>();
-    if core_object_file_paths.is_empty() {
-        return Err(format!("The provided --sysroot directory {:?} did not have any object files (.o)", sysroot_dir));
-    }
-
     let (
         app_object_files,
         kernel_objects_and_deps_files,
@@ -229,11 +215,7 @@ fn main() -> Result<(), String> {
         other_objects_and_deps_files.values().map(|(obj_direnty, _)| obj_direnty.path()),
         &kernel_prefix
     ).unwrap(); 
-    copy_files(
-        &output_objects_dir,
-        core_object_file_paths.into_iter(),
-        &kernel_prefix
-    ).unwrap(); 
+
 
 
     // Now we do the same kind of copy operation of crate dependency files, namely the .rlib and .rmeta files,
@@ -252,9 +234,28 @@ fn main() -> Result<(), String> {
         other_objects_and_deps_files.values().flat_map(|(_, deps)| deps.iter()),
         "",
     ).unwrap();
-    // Here we *could* optionally copy over the .rmeta/.rlib files from the sysroot, 
-    // but we choose not to because they are very large (tens of MBs) and aren't needed for future build steps.
-    // Also, this is debatably easier to do in the main Theseus Makefile.
+
+    // Here, if requested, we create the sysroot directory, containing the fundamental Rust libraries 
+    // that we ask cargo to build for us for Theseus's custom platform target
+    // Currently this comprises core, alloc, compiler_builtins, and rustc_std_workspace_core.
+    if let Some(output_sysroot_dir) = matches.opt_str("output-sysroot") {
+        fs::create_dir_all(&output_sysroot_dir).map_err(|e|
+            format!("Error creating output sysroot directory {:?}, {:?}", output_sysroot_dir, e)
+        )?;
+        let sysroot_files = other_objects_and_deps_files.iter()
+            .filter(|(crate_name, val)| {
+                crate_name.starts_with("core-") || 
+                crate_name.starts_with("compiler_builtins-") || 
+                crate_name.starts_with("rustc_std_workspace_core-") || 
+                crate_name.starts_with("alloc-")
+            })
+            .flat_map(|(_key, (_, deps))| deps.iter());
+        copy_files(
+            &output_sysroot_dir,
+            sysroot_files,
+            "",
+        ).unwrap();
+    }
 
     Ok(())
 }