Skip to content

Commit

Permalink
Update the stop sequence as per Omar's suggestion
Browse files Browse the repository at this point in the history
  • Loading branch information
insop committed Feb 4, 2024
1 parent 880600f commit c33f90f
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions dsp/modules/hf_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,14 +217,16 @@ def __init__(self, model, **kwargs):
if "inst" in self.model.lower() or "instruct" in model.lower():
self.use_inst_template = True

stop_default = "\n\n---"

self.kwargs = {
"temperature": 0.0,
"max_tokens": 512,
"top_p": 1,
"top_k": 20,
"repetition_penalty": 1,
"n": 1,
"stop": ["\n\n", "---", "[/INST]"],
"stop": stop_default if "stop" not in kwargs else kwargs["stop"],
**kwargs
}

Expand All @@ -247,7 +249,6 @@ def _generate(self, prompt, use_chat_api=False, **kwargs):
repetition_penalty = kwargs.get("repetition_penalty", 1)
prompt = f"[INST]{prompt}[/INST]" if self.use_inst_template else prompt


if use_chat_api:
url = f"{self.api_base}/chat/completions"
messages = [
Expand Down

0 comments on commit c33f90f

Please sign in to comment.