add new (empty) rust queue-scheduler

.dockerignore

@@ -1,6 +1,6 @@
 .git
 client/node_modules
-stroller/target
+*/target
 server/_build
 rundir
 sqldump*.gz

.gitignore

@@ -23,9 +23,11 @@ cronchecker.yaml
 queueworker.yaml
 tunnel.yaml
 
+# Rust projects build into target/
+/*/target
+
 # Files that get created on CI. We check that status is clean so need to list these.
 /client/node_modules
-/stroller/target
 /backend/_esy
 /backend/node_modules
 

Dockerfile

@@ -249,26 +249,12 @@ USER root
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.34.1
-
-RUN dpkgArch="$(dpkg --print-architecture)"; \
-    case "${dpkgArch##*-}" in \
-        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='0077ff9c19f722e2be202698c037413099e1188c0c233c12a2297bf18e9ff6e7' ;; \
-        armhf) rustArch='armv7-unknown-linux-gnueabihf'; rustupSha256='f139e5be4ea2db7ff151c122f5d24af3c587c4fc74a7414e262cb34403278ad3' ;; \
-        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='c7d5471e71a315134e7499af75eb177d1f574858f1c6b8e61b436702d671a4e2' ;; \
-        i386) rustArch='i686-unknown-linux-gnu'; rustupSha256='909ce4e2d0c9bf60ba5a85426c38cceb5ae77979ab2b1e354e76b9851b5ec5ed' ;; \
-        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
-    esac; \
-    url="https://static.rust-lang.org/rustup/archive/1.14.0/${rustArch}/rustup-init"; \
-    wget "$url"; \
-    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --default-toolchain $RUST_VERSION; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
+    RUST_VERSION=1.38.0
+
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_VERSION \
+  && rustup --version \
+  && cargo --version \
+  && rustc --version
 
 # install Rust dev tools
 RUN rustup component add clippy-preview rustfmt-preview

queue-scheduler/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "dark-queue-scheduler"
+version = "0.1.0"
+authors = ["Dark Inc <[email protected]>"]
+edition = "2018"
+
+[dependencies]

queue-scheduler/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

scripts/builder

@@ -60,6 +60,7 @@ else
   MOUNTS+=" --mount type=volume,src=dark_client_node_modules,dst=/home/dark/app/client/node_modules"
   MOUNTS+=" --mount type=volume,src=dark_client_lib,dst=/home/dark/app/client/lib"
   MOUNTS+=" --mount type=volume,src=dark_stroller_target,dst=/home/dark/app/stroller/target"
+  MOUNTS+=" --mount type=volume,src=dark_queue_scheduler_target,dst=/home/dark/app/queue-scheduler/target"
   MOUNTS+=" --mount type=volume,src=dark_rust_cargo,dst=/usr/local/cargo-home"
 
   if [[ -e "$HOME/.config/gcloud" ]]; then

scripts/support/build-server

@@ -55,6 +55,10 @@ def initial_stroller_compile():
   files = [ "stroller/src/main.rs" ]
   return compile(files)
 
+def initial_scheduler_compile():
+  files = [ "queue-scheduler/src/main.rs" ]
+  return compile(files)
+
 def run_server():
   exit = compile([ "scripts/support/runserver" ])
   print("--------------------------")
@@ -130,6 +134,7 @@ def main():
   compile_client = False
   compile_backend = False
   compile_stroller = False
+  compile_scheduler = False
 
   # Run the Dark webserver and keep the container open to serve it.
   # Useful for demoing without draining your battery. Will not recompile
@@ -142,6 +147,7 @@ def main():
       compile_backend = True
       compile_client = True
       compile_stroller = True
+      compile_scheduler = True
     elif f == "--compile-backend":
       compile = True
       compile_backend = True
@@ -151,6 +157,9 @@ def main():
     elif f == "--compile-stroller":
       compile = True
       compile_stroller = True
+    elif f == "--compile-scheduler":
+      compile = True
+      compile_scheduler = True
     elif f == "--watch":
       watch = True
     elif f == "--ci-serve":
@@ -173,6 +182,9 @@ def main():
     if compile_stroller and not initial_stroller_compile():
       print("Failed stroller build")
       success = False
+    if compile_scheduler and not initial_scheduler_compile():
+      print("Failed scheduler build")
+      success = False
 
 
     print("--------------------------")
@@ -191,6 +203,7 @@ def main():
   run("sudo chown dark:dark client/node_modules")
   run("sudo chown dark:dark client/lib")
   run("sudo chown dark:dark stroller/target")
+  run("sudo chown dark:dark queue-scheduler/target")
   run_or_fail("scripts/support/write-config-file")
   run_or_fail("scripts/support/allow-docker-access")
   run_or_fail("scripts/support/create-app-directories")

scripts/support/compile

@@ -155,12 +155,12 @@ def backend_test():
           + ci
           + " 2>&1")
 
-def stroller_build():
+def rust_build(name):
   global profile
 
   if os.environ.get("CI"):
     start = time.time()
-    result = run_backend(start, "cd stroller && cargo fmt -- --check")
+    result = run_backend(start, f"cd {name} && cargo fmt -- --check")
     if not result:
       return result
 
@@ -169,20 +169,20 @@ def stroller_build():
     build_flags = ""
 
   start = time.time()
-  build = "cd stroller && unbuffer cargo build" + build_flags
+  build = f"cd {name} && unbuffer cargo build"
   if profile:
     return run_backend(start, landmarks + build)
   else:
     return run_backend(start, build)
 
-def reload_stroller():
+def rust_reload(name):
   start = time.time()
-  return run_backend(start, "scripts/support/runstroller")
+  return run_backend(start, f"scripts/support/run-rust {name}")
 
-def stroller_test():
+def rust_test(name):
   start = time.time()
   result = run_test(start,
-      "cd stroller && unbuffer cargo test")
+      f"cd {name} && unbuffer cargo test")
   if not result:
       return result
 
@@ -192,8 +192,8 @@ def stroller_test():
   else:
       clippy_flags = ""
   return run_test(start,
-      "cd stroller"
-      + " && unbuffer cargo clippy --all-targets --bin=dark-stroller --tests -- " + clippy_flags)
+      f"cd {name}"
+      + f" && unbuffer cargo clippy --all-targets --bin=dark-{name} --tests -- " + clippy_flags)
 
 def reload_server():
   start = time.time()
@@ -220,6 +220,8 @@ class Should:
     self.backend_test = False
     self.stroller_build = False
     self.stroller_test = False
+    self.scheduler_build = False
+    self.scheduler_test = False
     self.npm_install = False
     self.client_build = False
     self.appsupport_build = False
@@ -228,6 +230,7 @@ class Should:
     self.reload_browser = False
     self.reload_server = False
     self.reload_stroller = False
+    self.reload_scheduler = False
     self.generate_etags = False
     self.shellcheck = []
 
@@ -286,17 +289,25 @@ def execute(should):
   if should.backend_test:
     if not backend_test(): success = False
 
-  # At the end do stroller, which isn't as important atm
+  # Rust projects
   if should.stroller_build:
-    if not stroller_build(): success = False
+    if not rust_build('stroller'): success = False
     should.stroller_test |= success
     should.reload_stroller |= success
-
   if should.reload_stroller:
-    if not reload_stroller(): success = False
-
+    if not rust_reload('stroller'): success = False
   if should.stroller_test:
-    if not stroller_test(): success = False
+    if not rust_test('stroller'): success = False
+
+  if should.scheduler_build:
+    if not rust_build('queue-scheduler'): success = False
+    should.scheduler_test |= success
+    should.reload_scheduler |= success
+  if should.reload_scheduler:
+    if not rust_reload('queue-scheduler'): success = False
+  if should.scheduler_test:
+    if not rust_test('queue-scheduler'): success = False
+
 
   if should.shellcheck != []:
     if not all([shellcheck(f) for f in should.shellcheck]):
@@ -330,14 +341,22 @@ def mark(should, f):
     should.generate_etags = True
 
   # Rust
+  # Touch this to rerun all rust programs
+  elif "scripts/support/run-rust" in f:
+    should.reload_stroller = True
+    should.reload_scheduler = True
+
   elif ("stroller/" in f) \
           and (("Cargo.toml" in f) \
                   or ("Cargo.lock" in f) \
                   or (".rs" in f)):
     should.stroller_build = True
-  # Touch this to rerun stroller
-  elif "scripts/support/runstroller" in f:
-    should.reload_stroller = True
+
+  elif ("queue-scheduler/" in f) \
+          and (("Cargo.toml" in f) \
+                  or ("Cargo.lock" in f) \
+                  or (".rs" in f)):
+    should.scheduler_build = True
 
   # JS
   elif "client/package.json" in f:
@@ -391,6 +410,7 @@ def ignore(filename):
             , "client/lib"
             , "build.ninja"
             , "stroller/target"
+            , "queue-scheduler/target"
             , ".bs.js"
             , ".bsdeps"
             , ".bsbuild"
@@ -407,6 +427,9 @@ def ignore(filename):
   # emacs thing
   if "/.#" in filename:
     return True
+  # vim thing
+  if filename[-1] == "~":
+    return True
   return False
 
 

scripts/support/run-rust

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+NAME="$1"
+
+if [[ -z "$NAME" ]] ; then
+  echo "Usage: $0 <name>"
+  exit 1
+fi
+
+if [[ -n $CI ]]; then
+  TARGET=release
+else
+  TARGET=debug
+fi
+
+BIN="$NAME/target/$TARGET/dark-$NAME"
+
+# Stop existing process (note that this makes the old process a zombie.
+# Doesn't matter though, as this only runs in dev).
+(ps aux | pgrep dark-$NAME | xargs kill -9 > /dev/null 2>&1) || true
+
+# if it hasn't been compiled yet, wait for it
+for ((i=1;i<=100;i++));
+do
+  if [[ ! -f "${BIN}" ]]; then
+    sleep 0.1
+  fi
+done
+
+if [[ -f "${BIN}" ]]; then
+  LOGS="${DARK_CONFIG_RUNDIR}/logs"
+  echo "Running $NAME"
+  "${BIN}" > "$LOGS/$NAME.log" 2>&1 &
+else
+  echo "Missing binary"
+  exit 1
+fi