End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 14.760727267175389,
-    "train_runtime": 146.8966,
+    "train_loss": 1.9396478868555278,
+    "train_runtime": 142.0942,
     "train_samples": 160,
-    "train_samples_per_second": 2.178,
-    "train_steps_per_second": 0.136
+    "train_samples_per_second": 2.252,
+    "train_steps_per_second": 0.141
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45bd292ad10f1ec9dd54b9f4321944a89c9536745e6bbd6b10c9c099ae4a2178
+oid sha256:106489dc517cf38317e2f716af4c550cacae8f805b91f198c1f41041f8e4d48b
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 14.760727267175389,
-    "train_runtime": 146.8966,
+    "train_loss": 1.9396478868555278,
+    "train_runtime": 142.0942,
     "train_samples": 160,
-    "train_samples_per_second": 2.178,
-    "train_steps_per_second": 0.136
+    "train_samples_per_second": 2.252,
+    "train_steps_per_second": 0.141
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 203.3125,
+      "completion_length": 249.5625,
       "epoch": 0.1,
-      "grad_norm": 21.456552505493164,
+      "grad_norm": 30.440135955810547,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 3.6251834090799093,
-      "reward_std": 0.5547963273493224,
-      "rewards/concensus_correctness_reward_func": 0.9458125047385693,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0,
+      "reward": 3.7942782919853926,
+      "reward_std": 0.5797093901783228,
+      "rewards/concensus_correctness_reward_func": 1.0694999918341637,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6402146760374308,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6646845547948033,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 0.8204062469303608,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 0.7632187479175627,
       "step": 2
     },
     {
-      "completion_length": 171.8125,
+      "completion_length": 154.0625,
       "epoch": 0.2,
-      "grad_norm": 12.347203254699707,
-      "kl": 1.7212362863501767,
+      "grad_norm": 1669.4842529296875,
+      "kl": 28.26861962856492,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0017,
-      "reward": 6.1263947784900665,
-      "reward_std": 0.16844019510608632,
-      "rewards/concensus_correctness_reward_func": 1.6204374954104424,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0283,
+      "reward": 6.498912572860718,
+      "reward_std": 0.5029855370521545,
+      "rewards/concensus_correctness_reward_func": 1.8760624900460243,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9768322855234146,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9860062636435032,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.2010000050067902,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1680937483906746,
       "step": 4
     },
     {
-      "completion_length": 202.6875,
+      "completion_length": 201.375,
       "epoch": 0.3,
-      "grad_norm": 20.570852279663086,
-      "kl": 0.14687257690820843,
+      "grad_norm": 71.76652526855469,
+      "kl": 2.3370895999250934,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0001,
-      "reward": 6.052525758743286,
-      "reward_std": 0.320869557079277,
-      "rewards/concensus_correctness_reward_func": 1.671499989926815,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0023,
+      "reward": 5.69296769797802,
+      "reward_std": 0.5531853784050327,
+      "rewards/concensus_correctness_reward_func": 1.51681250333786,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9914944730699062,
+      "rewards/question_recreation_reward_func": 0.9036864750087261,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0926562510430813,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.0068437438458204,
       "step": 6
     },
     {
-      "completion_length": 193.59375,
+      "completion_length": 152.625,
       "epoch": 0.4,
-      "grad_norm": 20.131040573120117,
-      "kl": 18.91134453518316,
+      "grad_norm": 499.4544982910156,
+      "kl": 9.919987747562118,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0189,
-      "reward": 5.503369979560375,
-      "reward_std": 0.8266189335845411,
-      "rewards/concensus_correctness_reward_func": 1.5803750036284328,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0099,
+      "reward": 6.4924532771110535,
+      "reward_std": 0.44198095612227917,
+      "rewards/concensus_correctness_reward_func": 1.9881874951533973,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.871963725425303,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.9540158621966839,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 0.8947812542319298,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.112749993801117,
       "step": 8
     },
     {
-      "completion_length": 143.84375,
+      "completion_length": 138.40625,
       "epoch": 0.5,
-      "grad_norm": 55.56569290161133,
-      "kl": 1.9424157487228513,
+      "grad_norm": 23.2545223236084,
+      "kl": 3.1933758644154295,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0019,
-      "reward": 7.324459671974182,
-      "reward_std": 0.36028057686053216,
-      "rewards/concensus_correctness_reward_func": 2.3316874876618385,
+      "loss": 0.0032,
+      "reward": 6.825431764125824,
+      "reward_std": 0.36719807982444763,
+      "rewards/concensus_correctness_reward_func": 2.1034999825060368,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.9767721369862556,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9331504553556442,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.187874998897314,
+      "rewards/xmlcount_reward_func": 1.210656251758337,
       "step": 10
     },
     {
-      "completion_length": 153.5,
+      "completion_length": 151.71875,
       "epoch": 0.6,
-      "grad_norm": 36.23406982421875,
-      "kl": 1.8086280468851328,
+      "grad_norm": 1900.021484375,
+      "kl": 55.82941412751097,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0018,
-      "reward": 5.904464989900589,
-      "reward_std": 0.34461313422070816,
-      "rewards/concensus_correctness_reward_func": 1.666749995201826,
+      "loss": 0.0558,
+      "reward": 5.894511699676514,
+      "reward_std": 0.5617045174585655,
+      "rewards/concensus_correctness_reward_func": 1.6966874860227108,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9170899763703346,
+      "rewards/question_recreation_reward_func": 0.8593241767957807,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1956249997019768,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.1353749968111515,
       "step": 12
     },
     {
-      "completion_length": 144.1875,
+      "completion_length": 138.78125,
       "epoch": 0.7,
-      "grad_norm": 640.2009887695312,
-      "kl": 78.81304021226242,
+      "grad_norm": 24.56352424621582,
+      "kl": 2.1570806360105053,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0788,
-      "reward": 6.339136034250259,
-      "reward_std": 0.33209400599298533,
-      "rewards/concensus_correctness_reward_func": 1.9186874888837337,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0022,
+      "reward": 5.838104248046875,
+      "reward_std": 0.3056765740038827,
+      "rewards/concensus_correctness_reward_func": 1.6357500031590462,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.982948549091816,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.9221042916178703,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.203125,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.1708749942481518,
       "step": 14
     },
     {
-      "completion_length": 159.59375,
+      "completion_length": 137.40625,
       "epoch": 0.8,
-      "grad_norm": 15.790488243103027,
-      "kl": 7.006550162099302,
+      "grad_norm": 1474.9375,
+      "kl": 17607.714312405325,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.007,
-      "reward": 6.786781519651413,
-      "reward_std": 0.28835287656693254,
-      "rewards/concensus_correctness_reward_func": 2.00212499499321,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 17.6077,
+      "reward": 6.172170788049698,
+      "reward_std": 0.763766348361969,
+      "rewards/concensus_correctness_reward_func": 1.8272499851882458,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9625628106296062,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8721082769334316,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.2283437512814999,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.160312496125698,
       "step": 16
     },
     {
-      "completion_length": 138.28125,
+      "completion_length": 139.28125,
       "epoch": 0.9,
-      "grad_norm": 10133141.0,
-      "kl": 147495.62073411373,
+      "grad_norm": 34.5722541809082,
+      "kl": 22.380590715736616,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 147.4956,
-      "reward": 6.325735956430435,
-      "reward_std": 0.4638390919790254,
-      "rewards/concensus_correctness_reward_func": 1.8305624723434448,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0224,
+      "reward": 7.503643155097961,
+      "reward_std": 0.5826042043045163,
+      "rewards/concensus_correctness_reward_func": 3.079812489449978,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9688296988606453,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.8400806300342083,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.151343747973442,
+      "rewards/xmlcount_reward_func": 1.1462500020861626,
       "step": 18
     },
     {
-      "completion_length": 143.40625,
+      "completion_length": 143.34375,
       "epoch": 1.0,
-      "grad_norm": 35.28290939331055,
-      "kl": 1.302146886009723,
+      "grad_norm": 60216.0859375,
+      "kl": 1664.6807740534423,
       "learning_rate": 0.0,
-      "loss": 0.0013,
-      "reward": 6.5682424902915955,
-      "reward_std": 0.63347500981763,
-      "rewards/concensus_correctness_reward_func": 2.0752499997615814,
+      "loss": 1.6647,
+      "reward": 6.0838969722390175,
+      "reward_std": 0.6379517484456301,
+      "rewards/concensus_correctness_reward_func": 1.8388749808073044,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9215237163007259,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9164595156908035,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.1027187518775463,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.0316875018179417,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 14.760727267175389,
-      "train_runtime": 146.8966,
-      "train_samples_per_second": 2.178,
-      "train_steps_per_second": 0.136
+      "train_loss": 1.9396478868555278,
+      "train_runtime": 142.0942,
+      "train_samples_per_second": 2.252,
+      "train_steps_per_second": 0.141
     }
   ],
   "logging_steps": 2,