update evaluate

aJupyter · Sep 18, 2024 · 05b658c · 05b658c
1 parent 091a936
commit 05b658c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 4 deletions.
diff --git a/evalueate/task/humaneval/humaneval.py b/evalueate/task/humaneval/humaneval.py
@@ -50,8 +50,6 @@ def generate_one(example, lang, tokenizer, model, args):
 def generate_main(args):
     model_name_or_path = args.model_name_or_path
     saved_path = args.output_path
-    temp_dir = args.temp_dir
-    os.makedirs(temp_dir, exist_ok=True)
 
     print("model", model_name_or_path)
     tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
@@ -94,4 +92,5 @@ def evaluate_only(args):
 if __name__ == '__main__':
     parser = HfArgumentParser((EvaluateArgs,))
     args = parser.parse_args_into_dataclasses()[0]
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
     generate_main(args)
diff --git a/evalueate/utils.py b/evalueate/utils.py
@@ -10,7 +10,7 @@ def extract_generation_code(example, lang_code):
 
     try:
         code_block: str = re.findall(f'```{lang.lower()}\n(.*?)```', output, re.DOTALL | re.IGNORECASE)[0]
-        assert code_block.startswith(prompt)
+        # assert code_block.startswith(prompt)
         generation = code_block[len(prompt):]
         example['generation'] = generation
 
@@ -34,4 +34,3 @@ def extract_generation_code(example, lang_code):
         'full_name': 'Java'
     }
 }
-