Skip to content

Commit

Permalink
GPU enablement and bug fixes
Browse files Browse the repository at this point in the history
Expose /dev/dri and /dev/kfd if they are present.

--instruct option was removed from llama.cpp so we need to do:

"-cnv", "-p", "You are a helpful assistant"

now. -cnv is conversation mode.

Also fixed a bug where return was missing.

Signed-off-by: Eric Curtin <[email protected]>
  • Loading branch information
ericcurtin committed Aug 22, 2024
1 parent 2b369d0 commit ed94877
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 6 deletions.
16 changes: 12 additions & 4 deletions ramalama.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ def main(args):
if os.path.exists(target):
wd = target
break

conman_args = [conman, "run",
"--rm",
"-it",
Expand All @@ -51,10 +52,17 @@ def main(args):
f"-v{home}:{home}",
"-v/tmp:/tmp",
f"-v{__file__}:/usr/bin/ramalama:ro",
f"-v{wd}:{syspath}:ro",
"-e", "RAMALAMA_HOST",
"-p", f"{host}:{port}",
"quay.io/ramalama/ramalama:latest", __file__] + args
f"-v{wd}:{syspath}:ro"]
if os.path.exists("/dev/dri"):
conman_args += ["--device", "/dev/dri"]

if os.path.exists("/dev/kfd"):
conman_args += ["--device", "/dev/kfd"]

conman_args += ["-e", "RAMALAMA_HOST",
"-p", f"{host}:{port}",
"quay.io/ramalama/ramalama:latest", __file__]
conman_args += args
if dryrun:
return print(*conman_args)

Expand Down
4 changes: 2 additions & 2 deletions ramalama/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ def pull_cli(store, args, port):
if transport == "ollama":
return ollama.pull(model, store)

ollama.pull(model, store)
return ollama.pull(model, store)


def push_cli(store, args, port):
Expand All @@ -222,7 +222,7 @@ def run_cli(store, args, port):

symlink_path = pull_cli(store, args, port)
exec_cmd(["llama-cli", "-m",
symlink_path, "--log-disable", "--instruct"])
symlink_path, "--log-disable", "-cnv", "-p", "You are a helpful assistant"])


def serve_cli(store, args, port):
Expand Down

0 comments on commit ed94877

Please sign in to comment.