add nltk

Dockerfile

@@ -29,7 +29,6 @@ RUN pip install packaging
 RUN pip install flash-attn==2.7.2.post1 --no-build-isolation
 COPY requirements.txt .
 RUN pip install -r requirements.txt
-RUN pip install -e .
 RUN python -m nltk.downloader punkt
 
 COPY open_instruct open_instruct

README.md

@@ -59,6 +59,12 @@ pip install -e .
 python -m nltk.downloader punkt
 ```
 
+* **Local installation with uv (preview)**: We are experimenting with using [uv](https://docs.astral.sh/uv/). You can install via
+```bash
+uv sync
+uv sync --extra compile # to install flash attention
+```
+
 
 * **Docker installation**: You can also use the Dockerfile to build a Docker image. You can build the image with the following command:
 
@@ -68,8 +74,22 @@ docker build . -t open_instruct_dev
 beaker_user=$(beaker account whoami --format json | jq -r '.[0].name')
 beaker image delete $beaker_user/open_instruct_dev 
 beaker image create open_instruct_dev -n open_instruct_dev -w ai2/$beaker_user
-# Optionally, you can build the base "cache" image with
+```
+
+Optionally you can build the base image with the following command:
+
+```bash
+docker build -f Dockerfile.base -t open_instruct_base .
+```
+
+* **Docker with uv**: You can also use the Dockerfile to build a Docker image with uv. You can build the image with the following command:
 
+```bash
+docker build -f Dockerfile.uv -t open_instruct_dev_uv .
+# if you are interally at AI2, you can create an image like this:
+beaker_user=$(beaker account whoami --format json | jq -r '.[0].name')
+beaker image delete $beaker_user/open_instruct_dev_uv 
+beaker image create open_instruct_dev_uv -n open_instruct_dev_uv -w ai2/$beaker_user
 ```
 
 If you are internally at AI2, you may launch experiments using our always-up-to-date auto-built image `nathanl/open_instruct_auto`.

pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "deepspeed>=0.16.0",
     "hf-transfer>=0.1.8",
     "matplotlib>=3.9.3",
+    "nltk>=3.9.1",
     "numpy<2",
     "packaging>=24.2",
     "peft>=0.13.2",

requirements.txt

@@ -64,6 +64,7 @@ interegular==0.3.3
 isort==5.13.2
 jinja2==3.1.5
 jiter==0.8.2
+joblib==1.4.2
 jsonschema==4.23.0
 jsonschema-specifications==2024.10.1
 kiwisolver==1.4.8
@@ -88,6 +89,7 @@ mypy-extensions==1.0.0
 nest-asyncio==1.6.0
 networkx==3.4.2
 ninja==1.11.1.3
+nltk==3.9.1
 numpy==1.26.4
 nvidia-cublas-cu12==12.1.3.1
 nvidia-cuda-cupti-cu12==12.1.105