Improve and fix tlibc and other C-compatibility layers (theseus-os#481

)

* `tlibc`: improve build process to support mixed Rust and C code components in a single library. This is needed to support varargs in exposed C functions like `printf`.

* Add C test app for print functions. Tested and working.

* `theseus_cargo`: remove the `.fingerprint` directory to force rustc to rebuild the final library output even though the original cargo invocation may have built it already.

applications/loadc/Cargo.toml

@@ -12,8 +12,8 @@ xmas-elf = { version = "0.6.2", git = "https://github.com/theseus-os/xmas-elf.gi
 rustc-demangle = "0.1.19"
 libc = { version = "0.2.107", default-features = false }
 
-[dependencies.terminal_print]
-path = "../../kernel/terminal_print"
+[dependencies.app_io]
+path = "../../kernel/app_io"
 
 [dependencies.path]
 path = "../../kernel/path"

applications/loadc/src/lib.rs

@@ -7,7 +7,7 @@
 
 #[macro_use] extern crate alloc;
 #[macro_use] extern crate log;
-#[macro_use] extern crate terminal_print;
+#[macro_use] extern crate app_io;
 extern crate getopts;
 extern crate fs_node;
 extern crate path;
@@ -77,6 +77,7 @@ fn rmain(matches: Matches) -> Result<c_int, String> {
     // Parse the file as an ELF executable
     let file_mp = file.as_mapping().map_err(|e| String::from(e))?;
     let byte_slice: &[u8] = file_mp.as_slice(0, file.size())?;
+
     let (mut segments, entry_point, _vaddr_offset, elf_file) = parse_and_load_elf_executable(byte_slice)?;
     debug!("Parsed ELF executable, moving on to overwriting relocations.");
 
@@ -351,7 +352,8 @@ fn overwrite_relocations(
         use xmas_elf::sections::SectionData::Rela64;
         if verbose_log { 
             trace!("Found Rela section name: {:?}, type: {:?}, target_sec_index: {:?}", 
-            sec.get_name(&elf_file), sec.get_type(), sec.info()); 
+                sec.get_name(&elf_file), sec.get_type(), sec.info()
+            ); 
         }
 
         let rela_sec_name = sec.get_name(&elf_file).unwrap();

c_test/Makefile

@@ -55,7 +55,7 @@ export override LDFLAGS += -nostdlib -nostartfiles
 ###
 ### TODO: figure out a way to remove all irrelevant relocation entries, at a minimum the `R_X86_64_NONE` and local relocation sections, such as the `.rela.eh_frame` section. 
 ### 
-%: %.c
+%: %.c $(TLIBC_A)
 	$(CROSS)gcc \
 		$(LDFLAGS) \
 		-O0 \

kernel/page_allocator/src/lib.rs

@@ -410,7 +410,6 @@ fn find_specific_chunk(
 							cursor_mut.move_prev(); // move the cursor back to the original chunk
 							let _removed_chunk = cursor_mut.replace_with(Wrapper::new_link(Chunk { pages: PageRange::new(*chunk.start(), new_end_page) }))
 								.expect("BUG: page_allocator failed to replace the current chunk while merging contiguous chunks.");
-							warn!("Untested: new_end_page: {:X?}, _removed_chunk: {:X?}", new_end_page, _removed_chunk);
 							return adjust_chosen_chunk(requested_page, num_pages, &chunk, ValueRefMut::RBTree(cursor_mut));
 						}
 					}

tlibc/Cargo.toml

@@ -34,6 +34,8 @@ path = "../kernel/app_io"
 ## tlibc into an .rlib file.
 [dependencies.panic_entry]
 path = "../kernel/panic_entry"
+[dependencies.heap]
+path = "../kernel/heap"
 
 
 [build-dependencies]

tlibc/build.sh

@@ -26,15 +26,23 @@ cargo clean
 ### with an automatic configuration that builds it to depend against pre-built Theseus crates.
 $THESEUS_CARGO_PATH/bin/theseus_cargo  --input $THESEUS_DEPS_DIR  build
 
-### Create a library archive (.a) file from all of the tlibc crate object files,
-### being sure to include the C code components.
-### Note: it's better to do a partial link, using `ld -r` below.
-ar -rcs ./target/x86_64-theseus/release/libtlibc.a  \
-    ./target/x86_64-theseus/release/deps/*.o        \
-    $(find ./target/x86_64-theseus/release/build/tlibc* -name "*.o")
+
+### The original output files built by cargo are invalid, so we remove them.
+rm -vf ./target/x86_64-theseus/release/libtlibc.a
+rm -vf ./target/x86_64-theseus/release/libtlibc.rlib
+rm -vf ./target/x86_64-theseus/release/libtlibc.d
+
+
+### tlibc's Cargo.toml now specifies the `staticlib` crate type, so the .a file is generated by rustc.
+### However, it exists in the target/.../deps/ file, and it has a hash appended to it.
+### We need to copy that file into the expected target folder and remove the hash.
+cp -vf ./target/x86_64-theseus/release/deps/libtlibc-*.a  ./target/x86_64-theseus/release/libtlibc.a
+
+
 
 ### Create a partially-linked object (.o) file from all of the tlibc crate object files,
 ### being sure to include the C code components.
+### Note that this will NOT include ALL Theseus .o dependencies, whereas the .a staticlib does.
 ld -r -o  ./target/x86_64-theseus/release/tlibc.o   \
     ./target/x86_64-theseus/release/deps/*.o        \
     $(find ./target/x86_64-theseus/release/build/tlibc* -name "*.o")
@@ -54,3 +62,34 @@ ld -r -o  ./target/x86_64-theseus/release/tlibc.o   \
 
 
 
+####################  Old stuff  ########################
+### Using `ar` to generate a static library from object files.
+### Note: this doesn't really work.
+# ar -rcs ./target/x86_64-theseus/release/libtlibc.a  \
+#     ./target/x86_64-theseus/release/deps/*.o        \
+#     $(find ./target/x86_64-theseus/release/build/tlibc* -name "*.o")
+
+
+### It also does NOT include the C components, so we need to include those as well.
+# ar -rcT ./target/x86_64-theseus/release/libtlibc_full.a              \
+#     ./target/x86_64-theseus/release/deps/*.a                         \
+#     ./target/x86_64-theseus/release/build/tlibc-*/out/libtlibc_c.a   
+
+
+# (                                                                        \
+#     cd ./target/x86_64-theseus/release  &&                               \
+#     ar -cqT libtlibc_full.a                                              \
+#         ./target/x86_64-theseus/release/./deps/*.a                                                       \
+#         ./target/x86_64-theseus/release/./build/tlibc-*/out/libtlibc_c.a                                 \
+#         &&                                                               \
+#     ar -M <(echo -e 'create libtlibc_full.a\naddlib libtlibc_full.a\nsave\nend')  \
+# )
+
+# ar -rcs ./target/x86_64-theseus/release/libtlibc_nested.a  \
+#     ./target/x86_64-theseus/release/deps/*.a             \
+#     $(find ./target/x86_64-theseus/release/build/tlibc* -name "*.o")
+# ### an alternative to the above
+# ar -rcs ./target/x86_64-theseus/release/libtlibc_nested_alt.a  \
+#     ./target/x86_64-theseus/release/deps/*.a          \
+#     ./target/x86_64-theseus/release/build/tlibc-*/out/libtlibc_c.a
+

tlibc/src/lib.rs

@@ -14,6 +14,8 @@
 
 // Needed for "staticlib" crate-type only
 extern crate panic_entry;
+extern crate heap;
+
 
 extern crate alloc;
 #[macro_use] extern crate log;

tools/theseus_cargo/src/main.rs

@@ -137,7 +137,10 @@ fn main() -> Result<(), String> {
                 }
             }
             _ => {
-                println!("Skipping potentially-redundant file with unexpected extension: {}", path.display());
+                println!("Removing potentially-redundant file with unexpected extension: {}", path.display());
+                fs::remove_file(&path).map_err(|e| 
+                    format!("Failed to remove potentially-redundant file with unexpected extension: {}. Error: {}", path.display(), e)
+                )?;
                 continue;
             }
         };
@@ -150,10 +153,40 @@ fn main() -> Result<(), String> {
                 format!("Failed to remove redundant crate file in out_dir: {}. Error: {}", path.display(), e)
             )?;
         } else {
-            // do nothing: keep the non-redundant file
+            // Here, do nothing. We must keep the non-redundant files,
+            // as they represent new dependencies that were not part of
+            // the original in-tree Theseus build.
         }
     }
 
+    // Here, remove the ".fingerprint/` directory, in order to force rustc
+    // to rebuild all artifacts for all of the modified rustc commands that we re-run below.
+    // Those fingerprint files are in the actual target directory, which is the parent directory of `out_dir`.
+    let target_dir = out_dir.parent().unwrap();
+    let mut fingerprint_dir_path = target_dir.to_path_buf();
+    fingerprint_dir_path.push(".fingerprint");
+    println!("--> Removing .fingerprint directory: {}", fingerprint_dir_path.display());
+    fs::remove_dir_all(&fingerprint_dir_path).map_err(|_e| 
+        format!("Failed to remove .fingerprint directory: {}. Error: {:?}", fingerprint_dir_path.display(), _e)
+    )?;
+
+        // if dir_entry.file_type().unwrap().is_file() {
+        //     match path.extension().and_then(|os_str| os_str.to_str()) {
+        //         Some("d") 
+        //         | Some("o")
+        //         | Some("a")
+        //         | Some(RMETA_FILE_EXTENSION)
+        //         | Some(RLIB_FILE_EXTENSION) => {
+        //             println!("Removing potentially-redundant target file: {}", path.display());
+        //             fs::remove_file(&path).map_err(|e| 
+        //                 format!("Failed to remove potentially-redundant target file: {}. Error: {}", path.display(), e)
+        //             )?;
+        //         }
+        //         _ => println!("Skipping existing file with unknown extension in target directory: {:?}", path.display()),
+        //     }
+        // }
+
+
     // Obtain the directory for the host system dependencies
     let mut host_deps_dir_path = PathBuf::from(&input_dir_path);
     host_deps_dir_path.push(&build_config.host_deps);