From 27cd693f32b87055561413e4c5ca091ed2d7c498 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Thu, 26 Dec 2024 16:43:58 +0100 Subject: [PATCH 01/20] builds example game with nix --- example-game/.gitignore | 1 - example-game/default.nix | 39 +++++++++++++++++++++++++++++++++ example-game/export_presets.cfg | 39 +++++++++++++++++++++++++++++++++ flake.nix | 6 ++++- nobodywho/default.nix | 5 +++-- 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 example-game/default.nix create mode 100644 example-game/export_presets.cfg diff --git a/example-game/.gitignore b/example-game/.gitignore index 5e99498..d7d166b 100644 --- a/example-game/.gitignore +++ b/example-game/.gitignore @@ -4,7 +4,6 @@ # Godot-specific ignores .import/ export.cfg -export_presets.cfg # Imported translations (automatically generated from CSV files) *.translation diff --git a/example-game/default.nix b/example-game/default.nix new file mode 100644 index 0000000..3620dde --- /dev/null +++ b/example-game/default.nix @@ -0,0 +1,39 @@ +{ nobodywho, stdenv, fetchurl, godot_4, godot_4-export-templates }: +let + model = fetchurl { + name = "gemma-2-2b-it-Q4_K_M.gguf"; + url = "https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q4_K_M.gguf"; + }; +in +stdenv.mkDerivation { + name = "nobodywho example game"; + src = ./.; + buildPhase = '' + # setup stuff godot needs: export templates + export HOME=$TMPDIR + mkdir -p $HOME/.local/share/godot/export_templates + ln -s ${godot_4-export-templates} $HOME/.local/share/godot/export_templates/4.3.stable + + # copy in gdextension stuff + rm ./nobodywho.gdextension + mkdir -p ./bin/ + cp ${nobodywho}/lib/libnobodywho.so ./bin/libnobodywho.so + cat << EOF > bin/nobodywho.gdextension + [configuration] + entry_symbol = "gdext_rust_init" + compatibility_minimum = 4.3 + reloadable = true + + [libraries] + linux.debug.x86_64 = "res://bin/libnobodywho.so" + linux.release.x86_64 = "res://bin/libnobodywho.so" + EOF + + # build game + mkdir -p $out + ${godot_4}/bin/godot4 --verbose --headless --export-debug "Linux" $out/game + ${godot_4}/bin/godot4 --verbose --headless --export-debug "Linux" $out/game + + cp ${model} $out/ + ''; +} diff --git a/example-game/export_presets.cfg b/example-game/export_presets.cfg new file mode 100644 index 0000000..59c32aa --- /dev/null +++ b/example-game/export_presets.cfg @@ -0,0 +1,39 @@ +[preset.0] + +name="Linux" +platform="Linux" +runnable=true +advanced_options=false +dedicated_server=false +custom_features="" +export_filter="all_resources" +include_filter="" +exclude_filter="" +export_path="out/New Game Project.x86_64" +encryption_include_filters="" +encryption_exclude_filters="" +encrypt_pck=false +encrypt_directory=false +script_export_mode=2 + +[preset.0.options] + +custom_template/debug="" +custom_template/release="" +debug/export_console_wrapper=1 +binary_format/embed_pck=false +texture_format/s3tc_bptc=true +texture_format/etc2_astc=false +binary_format/architecture="x86_64" +ssh_remote_deploy/enabled=false +ssh_remote_deploy/host="user@host_ip" +ssh_remote_deploy/port="22" +ssh_remote_deploy/extra_args_ssh="" +ssh_remote_deploy/extra_args_scp="" +ssh_remote_deploy/run_script="#!/usr/bin/env bash +export DISPLAY=:0 +unzip -o -q \"{temp_dir}/{archive_name}\" -d \"{temp_dir}\" +\"{temp_dir}/{exe_name}\" {cmd_args}" +ssh_remote_deploy/cleanup_script="#!/usr/bin/env bash +kill $(pgrep -x -f \"{temp_dir}/{exe_name} {cmd_args}\") +rm -rf \"{temp_dir}\"" diff --git a/flake.nix b/flake.nix index 6da522b..ce5e95d 100644 --- a/flake.nix +++ b/flake.nix @@ -8,9 +8,13 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = (import nixpkgs { system = system; }); + nobodywho = pkgs.callPackage ./nobodywho {}; + game = pkgs.callPackage ./example-game { inherit nobodywho; }; in { - packages.default = pkgs.callPackage ./nobodywho {}; + packages.default = nobodywho; + packages.game = game; + checks.default = game; devShells.default = import ./nobodywho/shell.nix { inherit pkgs; }; }); } diff --git a/nobodywho/default.nix b/nobodywho/default.nix index 6319af1..9148a47 100644 --- a/nobodywho/default.nix +++ b/nobodywho/default.nix @@ -10,8 +10,9 @@ rustPlatform.buildRustPackage { cargoLock = { lockFile = ./Cargo.lock; outputHashes = { - "gdextension-api-0.2.0" = "sha256-kIkOMwbO63pnmwG3nyM0gMtWhCKSMqz6fmd2nQ22wHg="; - "godot-0.1.3" = "sha256-T4HVbQ707obITx2dYAO8UYDM9Dvk6LMn6G3Ue8M1KqU="; + "gdextension-api-0.2.1" = "sha256-YkMbzObJGnmQa1XGT4ApRrfqAeOz7CktJrhYks8z0RY="; + "godot-0.2.0" = "sha256-jv0SsEiVrjCcOCVgc43gsTRk6HTImyRkRpZQRQD6rwc="; + "llama-cpp-2-0.1.86" = "sha256-Fe8WPO1NAISGGDkX5UWM8ubekYbnnAwEcKf0De5x9AQ="; }; }; env.LIBCLANG_PATH = "${libclang.lib}/lib/libclang.so"; From 5a028562e0db94da9260219da82b285bd36f10ce Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Mon, 6 Jan 2025 17:04:34 +0100 Subject: [PATCH 02/20] something works --- example-game/default.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/example-game/default.nix b/example-game/default.nix index 3620dde..72ed3b8 100644 --- a/example-game/default.nix +++ b/example-game/default.nix @@ -3,6 +3,12 @@ let model = fetchurl { name = "gemma-2-2b-it-Q4_K_M.gguf"; url = "https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q4_K_M.gguf"; + hash = "sha256-4K7oUGDxaPDy2Ec9fqQc4vMjDBvBN0hHUF6lmSiKd4c="; + }; + embedding_model = fetchurl { + name = "bge-small-en-v1.5-q8_0.gguf"; + url = "https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-q8_0.gguf"; + hash = "sha256-7Djo2hQllrqpExJK5QVQ3ihLaRa/WVd+8vDLlmDC9RQ="; }; in stdenv.mkDerivation { @@ -34,6 +40,7 @@ stdenv.mkDerivation { ${godot_4}/bin/godot4 --verbose --headless --export-debug "Linux" $out/game ${godot_4}/bin/godot4 --verbose --headless --export-debug "Linux" $out/game - cp ${model} $out/ + cp ${model} $out/gemma-2-2b-it-Q4_K_M.gguf + cp ${embedding_model} $out/bge-small-en-v1.5-q8_0.gguf ''; } From 754d5caa30ce98194b4b0126e9ac476621f84c15 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 10:39:17 +0100 Subject: [PATCH 03/20] print sparkles on success --- example-game/chat.gd | 2 +- example-game/embedding.gd | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/example-game/chat.gd b/example-game/chat.gd index c483570..4aea55a 100644 --- a/example-game/chat.gd +++ b/example-game/chat.gd @@ -10,4 +10,4 @@ func _ready(): # wait for the response var response = await response_finished - print("Got response: " + response) + print("✨ Got response: " + response) diff --git a/example-game/embedding.gd b/example-game/embedding.gd index 741e522..2a3e068 100644 --- a/example-game/embedding.gd +++ b/example-game/embedding.gd @@ -18,3 +18,5 @@ func _ready(): var low_similarity = cosine_similarity(irrelevant_embd, dragon_hill_embd) var high_similarity = cosine_similarity(dragon_hill_embd, dragon_hungry_embd) assert(low_similarity < high_similarity) + print("✨ embeddings completed") + From 3ed92956792267abe9ebc3bfce51746729aa33e9 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 10:48:37 +0100 Subject: [PATCH 04/20] make example game behave a bit more like an integration test --- example-game/chat.gd | 4 +++- example-game/embedding.gd | 7 ++++--- example-game/example.tscn | 4 +++- example-game/run_tests.gd | 8 ++++++++ 4 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 example-game/run_tests.gd diff --git a/example-game/chat.gd b/example-game/chat.gd index 4aea55a..e781d43 100644 --- a/example-game/chat.gd +++ b/example-game/chat.gd @@ -1,6 +1,6 @@ extends NobodyWhoChat -func _ready(): +func run_test(): # configure node model_node = get_node("../ChatModel") system_prompt = "You are an evil wizard. Always try to curse anyone who talks to you." @@ -11,3 +11,5 @@ func _ready(): # wait for the response var response = await response_finished print("✨ Got response: " + response) + if len(response) > 0: + return true diff --git a/example-game/embedding.gd b/example-game/embedding.gd index 2a3e068..7ffcbb5 100644 --- a/example-game/embedding.gd +++ b/example-game/embedding.gd @@ -1,6 +1,6 @@ extends NobodyWhoEmbedding -func _ready(): +func run_test(): # configure node self.model_node = get_node("../EmbeddingModel") @@ -17,6 +17,7 @@ func _ready(): # test similarity var low_similarity = cosine_similarity(irrelevant_embd, dragon_hill_embd) var high_similarity = cosine_similarity(dragon_hill_embd, dragon_hungry_embd) - assert(low_similarity < high_similarity) + var result = low_similarity < high_similarity + assert(result) print("✨ embeddings completed") - + return result diff --git a/example-game/example.tscn b/example-game/example.tscn index b19d6aa..3683ca7 100644 --- a/example-game/example.tscn +++ b/example-game/example.tscn @@ -1,6 +1,7 @@ -[gd_scene load_steps=3 format=3 uid="uid://qir8gkg0qx5w"] +[gd_scene load_steps=4 format=3 uid="uid://qir8gkg0qx5w"] [ext_resource type="Script" path="res://chat.gd" id="1_178kq"] +[ext_resource type="Script" path="res://run_tests.gd" id="1_mssk2"] [ext_resource type="Script" path="res://embedding.gd" id="2_rcagm"] [node name="Control" type="Control"] @@ -10,6 +11,7 @@ anchor_right = 1.0 anchor_bottom = 1.0 grow_horizontal = 2 grow_vertical = 2 +script = ExtResource("1_mssk2") [node name="ChatModel" type="NobodyWhoModel" parent="."] model_path = "res://gemma-2-2b-it-Q4_K_M.gguf" diff --git a/example-game/run_tests.gd b/example-game/run_tests.gd new file mode 100644 index 0000000..c6581dc --- /dev/null +++ b/example-game/run_tests.gd @@ -0,0 +1,8 @@ +extends Control + +func _ready() -> void: + print("👷 running tests...") + assert(await $NobodyWhoEmbedding.run_test()) + assert(await $NobodyWhoChat.run_test()) + print("✨ all tests complete") + get_tree().quit() From bbfb1bd3c93d8bb2b71a50cbf552062e81d59120 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 12:01:43 +0100 Subject: [PATCH 05/20] dont throw unreachable on recv channel death --- nobodywho/src/llm.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nobodywho/src/llm.rs b/nobodywho/src/llm.rs index 2a600f1..2adaab6 100644 --- a/nobodywho/src/llm.rs +++ b/nobodywho/src/llm.rs @@ -350,8 +350,9 @@ fn run_completion_worker_result( response.clear(); } - // This should be unreachable as the receiver loop only exits on error - unreachable!(); + // We can't really throw an error here, since the other end of our channels seem to have died + // but it's not `unreachable!()`, since we do end up here once the channels die. + Ok(()) // accept our fate } pub enum EmbeddingsOutput { Embedding(Vec), From b3d8cce3b2572e39bca56e1445d1c523150cc956 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 12:01:54 +0100 Subject: [PATCH 06/20] run integration test in flake --- flake.nix | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index ce5e95d..00a1ebc 100644 --- a/flake.nix +++ b/flake.nix @@ -7,14 +7,24 @@ outputs = { nixpkgs, flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: let - pkgs = (import nixpkgs { system = system; }); + pkgs = (import nixpkgs { + system = system; + config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ + "steam-run" + "steam-original" + ]; + }); nobodywho = pkgs.callPackage ./nobodywho {}; - game = pkgs.callPackage ./example-game { inherit nobodywho; }; + integration-test = pkgs.callPackage ./example-game { inherit nobodywho; }; + run-integration-test = pkgs.runCommand "checkgame" {} '' + cd ${integration-test} + ${pkgs.steam-run}/bin/steam-run ./game --headless + ''; in { packages.default = nobodywho; - packages.game = game; - checks.default = game; + packages.game = integration-test; + checks.default = run-integration-test; devShells.default = import ./nobodywho/shell.nix { inherit pkgs; }; }); } From 91c6032143744c45c8d3b6bf4d0889beacd73725 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 12:02:11 +0100 Subject: [PATCH 07/20] run flake check in github ci --- .github/workflows/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f25404..c5ccc39 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -185,6 +185,16 @@ jobs: - name: "Run unit tests" run: cargo test -- --nocapture --test-threads=1 working-directory: ./nobodywho + + nix-flake-check: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v27 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - run: nix build '.#game' + - run: nix flake check zip-distributable: From af6f19a67910704b6239a9eda7ea4e54e75f3419 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 14:34:17 +0100 Subject: [PATCH 08/20] dont run integration test in nix sandbox --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5ccc39..c99ba7e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -186,7 +186,7 @@ jobs: run: cargo test -- --nocapture --test-threads=1 working-directory: ./nobodywho - nix-flake-check: + integration-test: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -194,7 +194,7 @@ jobs: with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - run: nix build '.#game' - - run: nix flake check + - run: cd result && ./game --headless zip-distributable: From 4b166d531e0139493a5b89e213053a119fd95502 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 14:35:18 +0100 Subject: [PATCH 09/20] bump stuff --- flake.lock | 12 ++--- flake.nix | 1 + nobodywho/Cargo.lock | 105 +++++++++++++++++++++--------------------- nobodywho/default.nix | 38 ++++++++------- 4 files changed, 77 insertions(+), 79 deletions(-) diff --git a/flake.lock b/flake.lock index ec03c28..02324b7 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724479785, - "narHash": "sha256-pP3Azj5d6M5nmG68Fu4JqZmdGt4S4vqI5f8te+E/FTw=", + "lastModified": 1736012469, + "narHash": "sha256-/qlNWm/IEVVH7GfgAIyP6EsVZI6zjAx1cV5zNyrs+rI=", "owner": "nixos", "repo": "nixpkgs", - "rev": "d0e1602ddde669d5beb01aec49d71a51937ed7be", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 00a1ebc..23527c1 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,7 @@ config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "steam-run" "steam-original" + "steam-unwrapped" ]; }); nobodywho = pkgs.callPackage ./nobodywho {}; diff --git a/nobodywho/Cargo.lock b/nobodywho/Cargo.lock index 70f73d5..96e4535 100644 --- a/nobodywho/Cargo.lock +++ b/nobodywho/Cargo.lock @@ -81,9 +81,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "cc" -version = "1.2.2" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "jobserver", "libc", @@ -244,8 +244,8 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "godot" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" dependencies = [ "godot-core", "godot-macros", @@ -253,21 +253,21 @@ dependencies = [ [[package]] name = "godot-bindings" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" dependencies = [ "gdextension-api", ] [[package]] name = "godot-cell" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" [[package]] name = "godot-codegen" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" dependencies = [ "godot-bindings", "heck", @@ -279,8 +279,8 @@ dependencies = [ [[package]] name = "godot-core" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" dependencies = [ "glam", "godot-bindings", @@ -291,8 +291,8 @@ dependencies = [ [[package]] name = "godot-ffi" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" dependencies = [ "gensym", "godot-bindings", @@ -303,8 +303,8 @@ dependencies = [ [[package]] name = "godot-macros" -version = "0.2.0" -source = "git+https://github.com/godot-rust/gdext?branch=master#d32d569ed61ef8427470acdc41e8e708828fe412" +version = "0.2.2" +source = "git+https://github.com/godot-rust/gdext?branch=master#6a5d19d5f8e7563730ccd4738e46f398726b222a" dependencies = [ "godot-bindings", "proc-macro2", @@ -394,9 +394,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -416,9 +416,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.167" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" @@ -533,7 +533,7 @@ dependencies = [ "rusqlite", "serde", "sqlite-vec", - "thiserror 2.0.4", + "thiserror 2.0.9", ] [[package]] @@ -569,9 +569,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pkg-config" @@ -581,9 +581,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" dependencies = [ "proc-macro2", "syn", @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -677,24 +677,24 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "self_cell" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" +checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -703,9 +703,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -736,9 +736,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -756,11 +756,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.9", ] [[package]] @@ -776,9 +776,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", @@ -861,9 +861,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -872,13 +872,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -887,9 +886,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -897,9 +896,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -910,9 +909,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "which" diff --git a/nobodywho/default.nix b/nobodywho/default.nix index 9148a47..9e5dfb6 100644 --- a/nobodywho/default.nix +++ b/nobodywho/default.nix @@ -5,17 +5,31 @@ rustPlatform.buildRustPackage { pname = "nobody"; version = "0.0.0"; src = ./.; - nativeBuildInputs = [ llvmPackages_12.bintools cmake vulkan-headers vulkan-loader shaderc vulkan-tools mesa.drivers ]; - buildInputs = [ vulkan-loader vulkan-headers shaderc vulkan-tools mesa.drivers ]; + nativeBuildInputs = [ + llvmPackages_12.bintools + cmake + rustPlatform.bindgenHook + vulkan-headers + vulkan-loader + shaderc + vulkan-tools + mesa.drivers + ]; + buildInputs = [ + vulkan-loader + vulkan-headers + shaderc + vulkan-tools + mesa.drivers + ]; cargoLock = { lockFile = ./Cargo.lock; outputHashes = { "gdextension-api-0.2.1" = "sha256-YkMbzObJGnmQa1XGT4ApRrfqAeOz7CktJrhYks8z0RY="; - "godot-0.2.0" = "sha256-jv0SsEiVrjCcOCVgc43gsTRk6HTImyRkRpZQRQD6rwc="; + "godot-0.2.2" = "sha256-6q7BcQ/6WvzJdVmyAVGPMtuIDJFYKaRrkv3/JQBi11M="; "llama-cpp-2-0.1.86" = "sha256-Fe8WPO1NAISGGDkX5UWM8ubekYbnnAwEcKf0De5x9AQ="; }; }; - env.LIBCLANG_PATH = "${libclang.lib}/lib/libclang.so"; env.TEST_MODEL = fetchurl { name = "gemma-2-2b-it-Q5_K_M.gguf"; url = "https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q5_K_M.gguf"; @@ -27,22 +41,6 @@ rustPlatform.buildRustPackage { sha256 = "sha256-7Djo2hQllrqpExJK5QVQ3ihLaRa/WVd+8vDLlmDC9RQ="; }; - # See: https://hoverbear.org/blog/rust-bindgen-in-nix/ - preBuild = '' - # From: https://github.com/NixOS/nixpkgs/blob/1fab95f5190d087e66a3502481e34e15d62090aa/pkgs/applications/networking/browsers/firefox/common.nix#L247-L253 - # Set C flags for Rust's bindgen program. Unlike ordinary C - # compilation, bindgen does not invoke $CC directly. Instead it - # uses LLVM's libclang. To make sure all necessary flags are - # included we need to look in a few places. - export BINDGEN_EXTRA_CLANG_ARGS="$(< ${stdenv.cc}/nix-support/libc-crt1-cflags) \ - $(< ${stdenv.cc}/nix-support/libc-cflags) \ - $(< ${stdenv.cc}/nix-support/cc-cflags) \ - $(< ${stdenv.cc}/nix-support/libcxx-cxxflags) \ - ${lib.optionalString stdenv.cc.isClang "-idirafter ${stdenv.cc.cc}/lib/clang/${lib.getVersion stdenv.cc.cc}/include"} \ - ${lib.optionalString stdenv.cc.isGNU "-isystem ${stdenv.cc.cc}/include/c++/${lib.getVersion stdenv.cc.cc} -isystem ${stdenv.cc.cc}/include/c++/${lib.getVersion stdenv.cc.cc}/${stdenv.hostPlatform.config} -idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${lib.getVersion stdenv.cc.cc}/include"} \ - " - ''; - checkPhase = '' cargo test -- --test-threads=1 --nocapture ''; From 52427dbaf701ed48ae246c76401d41b463bfe1d0 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Tue, 7 Jan 2025 16:00:52 +0100 Subject: [PATCH 10/20] try with mesa drivers --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c99ba7e..50ac2a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,6 +190,10 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + - name: "Install deps from distro" + run: | + sudo apt-get update + sudo apt-get install -y mesa-vulkan-drivers - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} From 3f635b50115a328bd4613535bb5bc39c3e4a0f71 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 15:20:21 +0100 Subject: [PATCH 11/20] a debug step --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50ac2a0..1ed1ac2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -193,7 +193,9 @@ jobs: - name: "Install deps from distro" run: | sudo apt-get update - sudo apt-get install -y mesa-vulkan-drivers + sudo apt-get install -y mesa-vulkan-drivers vulkan-tools + - name: "Debug: list vulkan drivers" + run: "vulkaninfo --summary" - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} From 0fa6bae127fdd3d353daf8ef8227fdacb75419d8 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 15:42:14 +0100 Subject: [PATCH 12/20] add missing drivers from flake check --- .github/workflows/main.yml | 11 ++--------- flake.nix | 3 ++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ed1ac2..f638d3d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -186,21 +186,14 @@ jobs: run: cargo test -- --nocapture --test-threads=1 working-directory: ./nobodywho - integration-test: + nix-flake-check: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - name: "Install deps from distro" - run: | - sudo apt-get update - sudo apt-get install -y mesa-vulkan-drivers vulkan-tools - - name: "Debug: list vulkan drivers" - run: "vulkaninfo --summary" - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - run: nix build '.#game' - - run: cd result && ./game --headless + - run: nix flake check zip-distributable: diff --git a/flake.nix b/flake.nix index 23527c1..5a8014c 100644 --- a/flake.nix +++ b/flake.nix @@ -17,9 +17,10 @@ }); nobodywho = pkgs.callPackage ./nobodywho {}; integration-test = pkgs.callPackage ./example-game { inherit nobodywho; }; - run-integration-test = pkgs.runCommand "checkgame" {} '' + run-integration-test = pkgs.runCommand "checkgame" { nativeBuildInputs = with pkgs; [ mesa.drivers ]; } '' cd ${integration-test} ${pkgs.steam-run}/bin/steam-run ./game --headless + touch $out ''; in { From c391441cd69dd897bb556dd88e4a6a4f552ee9fb Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 15:50:45 +0100 Subject: [PATCH 13/20] use cachix binary cache for faster builds --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f638d3d..d0c5b85 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -193,6 +193,10 @@ jobs: - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} + - uses: cachix/cachix-action@vXX + with: + name: nobodywho + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix flake check From c1e50fedcf90ebbe7c3bb0dbdbb0ccd35f0d69f9 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 15:51:43 +0100 Subject: [PATCH 14/20] set proper cachix action version --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d0c5b85..28f57da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -193,7 +193,7 @@ jobs: - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - uses: cachix/cachix-action@vXX + - uses: cachix/cachix-action@v15 with: name: nobodywho authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' From 8f09621ee50d32b3ef3f0e8014ef0a42814ec113 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 15:57:23 +0100 Subject: [PATCH 15/20] well, screw caching anyway --- .github/workflows/main.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28f57da..a5a274a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -193,11 +193,16 @@ jobs: - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - uses: cachix/cachix-action@v15 - with: - name: nobodywho - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - run: nix flake check + - name: "Install deps from distro" + run: | + sudo apt-get update + sudo apt-get install -y vulkan-tools + - name: "build integration test" + run: "nix flake build '.#game'" + - name: "run integration test" + run: | + cd result + ./game --headless zip-distributable: From 6f765861faeed5d2b2df61e400010de8ca9eb96f Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 15:59:49 +0100 Subject: [PATCH 16/20] rename game -> integration-test --- .github/workflows/main.yml | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5a274a..55573dc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -198,7 +198,7 @@ jobs: sudo apt-get update sudo apt-get install -y vulkan-tools - name: "build integration test" - run: "nix flake build '.#game'" + run: "nix build '.#integration-test'" - name: "run integration test" run: | cd result diff --git a/flake.nix b/flake.nix index 5a8014c..0a6a23c 100644 --- a/flake.nix +++ b/flake.nix @@ -25,7 +25,7 @@ in { packages.default = nobodywho; - packages.game = integration-test; + packages.integration-test = integration-test; checks.default = run-integration-test; devShells.default = import ./nobodywho/shell.nix { inherit pkgs; }; }); From 6e1352a5ecfd97dc25690a470a6d8c4ccfe76443 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 16:20:33 +0100 Subject: [PATCH 17/20] go back to pure nix flake check --- .github/workflows/main.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55573dc..664bc25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,20 +190,9 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v27 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: "Install deps from distro" - run: | - sudo apt-get update - sudo apt-get install -y vulkan-tools - - name: "build integration test" - run: "nix build '.#integration-test'" - - name: "run integration test" - run: | - cd result - ./game --headless - + - uses: nixbuild/nix-quick-install-action@v27 + - name: "nix flake check" + run: "nix flake check -L" zip-distributable: needs: [cargo-build-linux, cargo-build-macos, cargo-build-windows] From 36781e844513d5217297e80a14dc0aaf11efda26 Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 16:37:37 +0100 Subject: [PATCH 18/20] run without steam-run --- flake.nix | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 0a6a23c..47c2c4c 100644 --- a/flake.nix +++ b/flake.nix @@ -7,19 +7,13 @@ outputs = { nixpkgs, flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: let - pkgs = (import nixpkgs { - system = system; - config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ - "steam-run" - "steam-original" - "steam-unwrapped" - ]; - }); + pkgs = (import nixpkgs { system = system; }); nobodywho = pkgs.callPackage ./nobodywho {}; integration-test = pkgs.callPackage ./example-game { inherit nobodywho; }; run-integration-test = pkgs.runCommand "checkgame" { nativeBuildInputs = with pkgs; [ mesa.drivers ]; } '' cd ${integration-test} - ${pkgs.steam-run}/bin/steam-run ./game --headless + export HOME=$TMPDIR + ./game --headless touch $out ''; in From ead833f762743ddc5db4ba5b7e81c30d55cd4c9f Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 16:50:08 +0100 Subject: [PATCH 19/20] patch interpreter --- example-game/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example-game/default.nix b/example-game/default.nix index 72ed3b8..bee60ce 100644 --- a/example-game/default.nix +++ b/example-game/default.nix @@ -42,5 +42,8 @@ stdenv.mkDerivation { cp ${model} $out/gemma-2-2b-it-Q4_K_M.gguf cp ${embedding_model} $out/bge-small-en-v1.5-q8_0.gguf + + # Patch binaries. + patchelf --set-interpreter $(cat $NIX_CC/nix-support/dynamic-linker) $out/game ''; } From febeea515ced81e1c5187da151219cc479c4753b Mon Sep 17 00:00:00 2001 From: AsbjornOlling Date: Wed, 8 Jan 2025 16:50:39 +0100 Subject: [PATCH 20/20] use same model in unit tests and integration tests --- nobodywho/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nobodywho/default.nix b/nobodywho/default.nix index 9e5dfb6..7965670 100644 --- a/nobodywho/default.nix +++ b/nobodywho/default.nix @@ -31,9 +31,9 @@ rustPlatform.buildRustPackage { }; }; env.TEST_MODEL = fetchurl { - name = "gemma-2-2b-it-Q5_K_M.gguf"; - url = "https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q5_K_M.gguf"; - sha256 = "1njh254wpsg2j4wi686zabg63n42fmkgdmf9v3cl1zbydybdardy"; + name = "gemma-2-2b-it-Q4_K_M.gguf"; + url = "https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q4_K_M.gguf"; + hash = "sha256-4K7oUGDxaPDy2Ec9fqQc4vMjDBvBN0hHUF6lmSiKd4c="; }; env.TEST_EMBEDDINGS_MODEL = fetchurl { name = "bge-small-en-v1.5-q8_0.gguf";