Suport BigCode model (stanford-crfm#1506)

src/helm/benchmark/run_expander.py

@@ -763,6 +763,7 @@ class TokenizerRunExpander(ScenarioSpecRunExpander):
         "AlephAlpha/luminous-supreme": ["AlephAlpha/luminous-supreme"],
         "AlephAlpha/luminous-world": ["AlephAlpha/luminous-world"],
         "huggingface/santacoder": ["bigcode/santacoder"],
+        "huggingface/large-model": ["bigcode/large-model"],
     }
     model_tags_and_tokenizers = [
         (GPT2_TOKENIZER_TAG, "huggingface/gpt2"),

src/helm/benchmark/static/schema.yaml

@@ -119,6 +119,11 @@ models:
     description: SantaCoder (1.1B parameters) model trained on the Python, Java, and JavaScript subset of The Stack (v1.1) ([model card](https://huggingface.co/bigcode/santacoder)).
     creator_organization: BigCode
     access: open
+  - name: huggingface/large-model
+    display_name: BigCode's Large Model (15.5B)
+    description: BigCode's Large Model (15.5B parameters) trained on The Stack (v1.2) ([model card](https://huggingface.co/bigcode/large-model)).
+    creator_organization: BigCode
+    access: open
 
   # Cohere
   - name: cohere/xlarge-20220609

src/helm/benchmark/window_services/bigcode_large_model_window_service.py

@@ -0,0 +1,27 @@
+from .local_window_service import LocalWindowService
+from .tokenizer_service import TokenizerService
+
+
+class BigCodeLargeModelWindowService(LocalWindowService):
+    def __init__(self, service: TokenizerService):
+        super().__init__(service)
+
+    @property
+    def max_sequence_length(self) -> int:
+        return 8192
+
+    @property
+    def max_request_length(self) -> int:
+        return self.max_sequence_length
+
+    @property
+    def end_of_text_token(self) -> str:
+        return "<|endoftext|>"
+
+    @property
+    def tokenizer_name(self) -> str:
+        return "bigcode/large-model"
+
+    @property
+    def prefix_token(self) -> str:
+        return self.end_of_text_token

src/helm/benchmark/window_services/window_service_factory.py

@@ -24,6 +24,7 @@
 from .huggingface_window_service import HuggingFaceWindowService
 from .ice_window_service import ICEWindowService
 from .santacoder_window_service import SantaCoderWindowService
+from .bigcode_large_model_window_service import BigCodeLargeModelWindowService
 from .gpt2_window_service import GPT2WindowService
 from .gptj_window_service import GPTJWindowService
 from .gptneox_window_service import GPTNeoXWindowService
@@ -86,6 +87,8 @@ def get_window_service(model_name: str, service: TokenizerService) -> WindowServ
             window_service = AnthropicWindowService(service)
         elif engine == "santacoder":
             window_service = SantaCoderWindowService(service)
+        elif engine == "large-model":
+            window_service = BigCodeLargeModelWindowService(service)
         elif model_name == "huggingface/gpt2":
             window_service = GPT2WindowService(service)
         elif model_name == "together/bloom":

src/helm/proxy/clients/huggingface_client.py

@@ -133,6 +133,10 @@ def get_model_server_instance(self, model) -> HuggingFaceServer:
                 self.model_server_instances[model] = HuggingFaceServer(
                     HuggingFaceModelConfig.from_string("bigcode/santacoder")
                 )
+            elif model == "huggingface/large-model":
+                self.model_server_instances[model] = HuggingFaceServer(
+                    HuggingFaceModelConfig.from_string("bigcode/large-model")
+                )
             else:
                 raise Exception(f"Unknown HuggingFace model: {model}")
 

src/helm/proxy/clients/huggingface_tokenizer.py

@@ -76,6 +76,8 @@ def load_tokenizer(hf_tokenizer_name: str, revision: Optional[str] = None):
                     hf_tokenizer_name = "google/flan-t5-xxl"
                 elif tokenizer_name == "bigcode/santacoder":
                     hf_tokenizer_name = "bigcode/santacoder"
+                elif tokenizer_name == "bigcode/large-model":
+                    hf_tokenizer_name = "bigcode/large-model"
                 else:
                     raise ValueError(f"Unsupported HuggingFace tokenizer: {tokenizer_name}")
 

src/helm/proxy/models.py

@@ -374,6 +374,14 @@ def engine(self) -> str:
         "JavaScript subset of The Stack (v1.1).",
         tags=[CODE_MODEL_TAG],
     ),
+    Model(
+        group="huggingface",
+        creator_organization="BigCode",
+        name="huggingface/large-model",
+        display_name="BigCode's Large Model (15.5B)",
+        description="BigCode's Large Model (15.5B) trained on The Stack (v1.2)",
+        tags=[CODE_MODEL_TAG],
+    ),
     # Google
     Model(
         group="together",