Support batch eval for trutufulqa mc setup.

yifan1130 · Oct 19, 2023 · 677cbd0 · 677cbd0
1 parent e11dd37
commit 677cbd0
Show file tree

Hide file tree

Showing 3 changed files with 191 additions and 242 deletions.
diff --git a/eval/truthfulqa/metrics.py b/eval/truthfulqa/metrics.py
@@ -48,7 +48,7 @@ def MC_calcs(tag, frame, idx, scores_true, scores_false, ref_true, ref_best):
     frame.loc[idx, '{0} MC2'.format(tag)] = sum(probs_true)
 
 
-def run_end2end_GPT3(model_key, tag, engine, frame, info=False):
+def run_gpt3_classifier_eval(model_key, tag, engine, frame, info=False):
 
     """
     Uses a fine-tuned GPT-3 model to predict human evaluations of truth and infomativeness