From 648af26073058df66a98b58f98e37e2e4d260974 Mon Sep 17 00:00:00 2001
From: Eric Wang <eric.james.wang@gmail.com>
Date: Tue, 14 Mar 2023 08:51:30 -0700
Subject: [PATCH] update hyperparams

---
 finetune.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/finetune.py b/finetune.py
index 10876d68..673b3f95 100644
--- a/finetune.py
+++ b/finetune.py
@@ -62,8 +62,8 @@ def generate_prompt(data_point):
 MICRO_BATCH_SIZE = 4  # this could actually be 5 but i like powers of 2
 BATCH_SIZE = 128
 GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
-EPOCHS = 3  # from the result
-LEARNING_RATE = 2e-5  # also from the result
+EPOCHS = 1  # we don't need 3 tbh
+LEARNING_RATE = 3e-4  # the karpathy constant
 CUTOFF_LEN = 256  # 256 accounts for about 96% of the data
 
 data = data.shuffle().map(