End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.07148093223804608,
-    "train_runtime": 148.1766,
+    "train_loss": 14.760727267175389,
+    "train_runtime": 146.8966,
     "train_samples": 160,
-    "train_samples_per_second": 2.16,
-    "train_steps_per_second": 0.135
+    "train_samples_per_second": 2.178,
+    "train_steps_per_second": 0.136
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba88c857e7a309944e92d6240014d9aa5ba2dbf08141242c42d24a04ae1720de
+oid sha256:45bd292ad10f1ec9dd54b9f4321944a89c9536745e6bbd6b10c9c099ae4a2178
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.07148093223804608,
-    "train_runtime": 148.1766,
+    "train_loss": 14.760727267175389,
+    "train_runtime": 146.8966,
     "train_samples": 160,
-    "train_samples_per_second": 2.16,
-    "train_steps_per_second": 0.135
+    "train_samples_per_second": 2.178,
+    "train_steps_per_second": 0.136
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 300.09375,
+      "completion_length": 203.3125,
       "epoch": 0.1,
-      "grad_norm": 23.860929489135742,
+      "grad_norm": 21.456552505493164,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 4.236713282763958,
-      "reward_std": 0.9856820134446025,
-      "rewards/concensus_correctness_reward_func": 1.2228749878704548,
-      "rewards/consensus_reward_func": 1.1875,
+      "reward": 3.6251834090799093,
+      "reward_std": 0.5547963273493224,
+      "rewards/concensus_correctness_reward_func": 0.9458125047385693,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7189320279285312,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6402146760374308,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.140625,
-      "rewards/xmlcount_reward_func": 0.8417812511324883,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 0.8204062469303608,
       "step": 2
     },
     {
-      "completion_length": 172.6875,
+      "completion_length": 171.8125,
       "epoch": 0.2,
-      "grad_norm": 74.83126831054688,
-      "kl": 0.7343315020407317,
+      "grad_norm": 12.347203254699707,
+      "kl": 1.7212362863501767,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0007,
-      "reward": 5.874257683753967,
-      "reward_std": 0.5174596148426645,
-      "rewards/concensus_correctness_reward_func": 1.5709999985992908,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0017,
+      "reward": 6.1263947784900665,
+      "reward_std": 0.16844019510608632,
+      "rewards/concensus_correctness_reward_func": 1.6204374954104424,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9224452041089535,
+      "rewards/question_recreation_reward_func": 0.9768322855234146,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.1151874959468842,
+      "rewards/xmlcount_reward_func": 1.2010000050067902,
       "step": 4
     },
     {
-      "completion_length": 196.25,
+      "completion_length": 202.6875,
       "epoch": 0.3,
-      "grad_norm": 19.436935424804688,
-      "kl": 503.66224323713686,
+      "grad_norm": 20.570852279663086,
+      "kl": 0.14687257690820843,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.5037,
-      "reward": 6.5185030698776245,
-      "reward_std": 0.25980229781998787,
-      "rewards/concensus_correctness_reward_func": 1.9238749966025352,
+      "loss": 0.0001,
+      "reward": 6.052525758743286,
+      "reward_std": 0.320869557079277,
+      "rewards/concensus_correctness_reward_func": 1.671499989926815,
       "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9977218806743622,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9914944730699062,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1750312522053719,
+      "rewards/xmlcount_reward_func": 1.0926562510430813,
       "step": 6
     },
     {
-      "completion_length": 147.90625,
+      "completion_length": 193.59375,
       "epoch": 0.4,
-      "grad_norm": 27.34542465209961,
-      "kl": 33.45032201614231,
+      "grad_norm": 20.131040573120117,
+      "kl": 18.91134453518316,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0335,
-      "reward": 6.338675796985626,
-      "reward_std": 0.7246713064378127,
-      "rewards/concensus_correctness_reward_func": 1.867374992929399,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0189,
+      "reward": 5.503369979560375,
+      "reward_std": 0.8266189335845411,
+      "rewards/concensus_correctness_reward_func": 1.5803750036284328,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8552383631467819,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.871963725425303,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1785625033080578,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 0.8947812542319298,
       "step": 8
     },
     {
-      "completion_length": 129.46875,
+      "completion_length": 143.84375,
       "epoch": 0.5,
-      "grad_norm": 30.70717430114746,
-      "kl": 47.86718482570723,
+      "grad_norm": 55.56569290161133,
+      "kl": 1.9424157487228513,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0479,
-      "reward": 7.684082388877869,
-      "reward_std": 0.26071762200444937,
-      "rewards/concensus_correctness_reward_func": 2.5246249958872795,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0019,
+      "reward": 7.324459671974182,
+      "reward_std": 0.36028057686053216,
+      "rewards/concensus_correctness_reward_func": 2.3316874876618385,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.9172698333859444,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.9767721369862556,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.2109375,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.187874998897314,
       "step": 10
     },
     {
-      "completion_length": 138.21875,
+      "completion_length": 153.5,
       "epoch": 0.6,
-      "grad_norm": 519.1696166992188,
-      "kl": 28.34276645909995,
+      "grad_norm": 36.23406982421875,
+      "kl": 1.8086280468851328,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0283,
-      "reward": 6.716021865606308,
-      "reward_std": 0.28479388065170497,
-      "rewards/concensus_correctness_reward_func": 1.9936249740421772,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0018,
+      "reward": 5.904464989900589,
+      "reward_std": 0.34461313422070816,
+      "rewards/concensus_correctness_reward_func": 1.666749995201826,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8473968785256147,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.9170899763703346,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.1875,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1956249997019768,
       "step": 12
     },
     {
-      "completion_length": 150.1875,
+      "completion_length": 144.1875,
       "epoch": 0.7,
-      "grad_norm": 130.59896850585938,
-      "kl": 11.308747926028445,
+      "grad_norm": 640.2009887695312,
+      "kl": 78.81304021226242,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0113,
-      "reward": 6.695678576827049,
-      "reward_std": 0.43522001011297107,
-      "rewards/concensus_correctness_reward_func": 2.078312497586012,
+      "loss": 0.0788,
+      "reward": 6.339136034250259,
+      "reward_std": 0.33209400599298533,
+      "rewards/concensus_correctness_reward_func": 1.9186874888837337,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9712723717093468,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.982948549091816,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.0992187485098839,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.203125,
       "step": 14
     },
     {
-      "completion_length": 144.0,
+      "completion_length": 159.59375,
       "epoch": 0.8,
-      "grad_norm": 5816.69189453125,
-      "kl": 87.2975728274323,
+      "grad_norm": 15.790488243103027,
+      "kl": 7.006550162099302,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0873,
-      "reward": 6.687029004096985,
-      "reward_std": 0.369557810947299,
-      "rewards/concensus_correctness_reward_func": 1.9977499954402447,
+      "loss": 0.007,
+      "reward": 6.786781519651413,
+      "reward_std": 0.28835287656693254,
+      "rewards/concensus_correctness_reward_func": 2.00212499499321,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.947091531008482,
+      "rewards/question_recreation_reward_func": 0.9625628106296062,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2109375,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.2283437512814999,
       "step": 16
     },
     {
-      "completion_length": 137.1875,
+      "completion_length": 138.28125,
       "epoch": 0.9,
-      "grad_norm": 37.66142654418945,
-      "kl": 1.5667426733998582,
+      "grad_norm": 10133141.0,
+      "kl": 147495.62073411373,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0016,
-      "reward": 5.787939324975014,
-      "reward_std": 1.1536127398721874,
-      "rewards/concensus_correctness_reward_func": 1.6428125016391277,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 147.4956,
+      "reward": 6.325735956430435,
+      "reward_std": 0.4638390919790254,
+      "rewards/concensus_correctness_reward_func": 1.8305624723434448,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8834080211818218,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9688296988606453,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.15234375,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.151343747973442,
       "step": 18
     },
     {
-      "completion_length": 156.96875,
+      "completion_length": 143.40625,
       "epoch": 1.0,
-      "grad_norm": 19.40934181213379,
-      "kl": 0.5885741473175585,
+      "grad_norm": 35.28290939331055,
+      "kl": 1.302146886009723,
       "learning_rate": 0.0,
-      "loss": 0.0006,
-      "reward": 6.706097096204758,
-      "reward_std": 0.4787634233944118,
-      "rewards/concensus_correctness_reward_func": 2.11218748614192,
+      "loss": 0.0013,
+      "reward": 6.5682424902915955,
+      "reward_std": 0.63347500981763,
+      "rewards/concensus_correctness_reward_func": 2.0752499997615814,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9290658328682184,
+      "rewards/question_recreation_reward_func": 0.9215237163007259,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.149218751117587,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.1027187518775463,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.07148093223804608,
-      "train_runtime": 148.1766,
-      "train_samples_per_second": 2.16,
-      "train_steps_per_second": 0.135
+      "train_loss": 14.760727267175389,
+      "train_runtime": 146.8966,
+      "train_samples_per_second": 2.178,
+      "train_steps_per_second": 0.136
     }
   ],
   "logging_steps": 2,