| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 375, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 270.8035888671875, | |
| "epoch": 0.0026666666666666666, | |
| "grad_norm": 0.7334129323891274, | |
| "kl": 0.0001461505889892578, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0712, | |
| "reward": 0.6089551746845245, | |
| "reward_std": 0.1534397415816784, | |
| "rewards/length_reward": 0.06294643878936768, | |
| "rewards/similarity_reward": 0.546008750796318, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 248.49555206298828, | |
| "epoch": 0.005333333333333333, | |
| "grad_norm": 0.9035152344267738, | |
| "kl": 0.00016069412231445312, | |
| "learning_rate": 2e-06, | |
| "loss": 0.06, | |
| "reward": 0.5894998908042908, | |
| "reward_std": 0.17237309366464615, | |
| "rewards/length_reward": 0.061160728335380554, | |
| "rewards/similarity_reward": 0.528339147567749, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 271.5848388671875, | |
| "epoch": 0.008, | |
| "grad_norm": 0.8948061881977104, | |
| "kl": 0.00017070770263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0442, | |
| "reward": 0.6276047825813293, | |
| "reward_std": 0.15094101428985596, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.5561762154102325, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 257.46429443359375, | |
| "epoch": 0.010666666666666666, | |
| "grad_norm": 0.7848935176364024, | |
| "kl": 0.00016450881958007812, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0583, | |
| "reward": 0.5882956981658936, | |
| "reward_std": 0.1704816073179245, | |
| "rewards/length_reward": 0.06473215110599995, | |
| "rewards/similarity_reward": 0.5235635340213776, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 240.4821548461914, | |
| "epoch": 0.013333333333333334, | |
| "grad_norm": 0.7884926165963533, | |
| "kl": 0.000179290771484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0444, | |
| "reward": 0.6010282635688782, | |
| "reward_std": 0.10747816786170006, | |
| "rewards/length_reward": 0.058482151478528976, | |
| "rewards/similarity_reward": 0.5425460636615753, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 271.5758972167969, | |
| "epoch": 0.016, | |
| "grad_norm": 0.7176003677981444, | |
| "kl": 0.0001480579376220703, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1772, | |
| "reward": 0.632064938545227, | |
| "reward_std": 0.13313144445419312, | |
| "rewards/length_reward": 0.06562501192092896, | |
| "rewards/similarity_reward": 0.5664399564266205, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 248.83483123779297, | |
| "epoch": 0.018666666666666668, | |
| "grad_norm": 0.583690210979672, | |
| "kl": 0.000118255615234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.01, | |
| "reward": 0.6557432115077972, | |
| "reward_std": 0.09923132508993149, | |
| "rewards/length_reward": 0.0714285783469677, | |
| "rewards/similarity_reward": 0.584314614534378, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 240.2991180419922, | |
| "epoch": 0.021333333333333333, | |
| "grad_norm": 0.8285932775348754, | |
| "kl": 0.00018167495727539062, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0352, | |
| "reward": 0.5436868965625763, | |
| "reward_std": 0.19458478689193726, | |
| "rewards/length_reward": 0.053571440279483795, | |
| "rewards/similarity_reward": 0.4901154637336731, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 274.74554443359375, | |
| "epoch": 0.024, | |
| "grad_norm": 0.6297183637280267, | |
| "kl": 0.00014972686767578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0198, | |
| "reward": 0.6230277419090271, | |
| "reward_std": 0.15007304400205612, | |
| "rewards/length_reward": 0.06562501192092896, | |
| "rewards/similarity_reward": 0.5574026703834534, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 271.2991180419922, | |
| "epoch": 0.02666666666666667, | |
| "grad_norm": 0.6908976518587052, | |
| "kl": 0.00015354156494140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0383, | |
| "reward": 0.6355253756046295, | |
| "reward_std": 0.16409822553396225, | |
| "rewards/length_reward": 0.0647321492433548, | |
| "rewards/similarity_reward": 0.5707932114601135, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 263.7901916503906, | |
| "epoch": 0.029333333333333333, | |
| "grad_norm": 0.7190922283358612, | |
| "kl": 0.00019359588623046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.039, | |
| "reward": 0.6213563680648804, | |
| "reward_std": 0.14211475104093552, | |
| "rewards/length_reward": 0.07053572311997414, | |
| "rewards/similarity_reward": 0.5508206188678741, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 268.21429443359375, | |
| "epoch": 0.032, | |
| "grad_norm": 0.7674287367860577, | |
| "kl": 0.0001621246337890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0666, | |
| "reward": 0.6959913372993469, | |
| "reward_std": 0.13710426539182663, | |
| "rewards/length_reward": 0.07098215445876122, | |
| "rewards/similarity_reward": 0.6250091791152954, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 308.23216247558594, | |
| "epoch": 0.034666666666666665, | |
| "grad_norm": 0.6961227618622208, | |
| "kl": 0.00017309188842773438, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1339, | |
| "reward": 0.6554811000823975, | |
| "reward_std": 0.14746695756912231, | |
| "rewards/length_reward": 0.06383929960429668, | |
| "rewards/similarity_reward": 0.5916417837142944, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 255.1696548461914, | |
| "epoch": 0.037333333333333336, | |
| "grad_norm": 0.6790296137610172, | |
| "kl": 0.00018310546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0102, | |
| "reward": 0.639388918876648, | |
| "reward_std": 0.13504448905587196, | |
| "rewards/length_reward": 0.07232143729925156, | |
| "rewards/similarity_reward": 0.5670674443244934, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 288.5982208251953, | |
| "epoch": 0.04, | |
| "grad_norm": 0.5592059952702082, | |
| "kl": 0.00015926361083984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0083, | |
| "reward": 0.6401915550231934, | |
| "reward_std": 0.10427659377455711, | |
| "rewards/length_reward": 0.061607155948877335, | |
| "rewards/similarity_reward": 0.5785843580961227, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 256.5669708251953, | |
| "epoch": 0.042666666666666665, | |
| "grad_norm": 0.6493017975612209, | |
| "kl": 0.0001373291015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0011, | |
| "reward": 0.6510922610759735, | |
| "reward_std": 0.12598292529582977, | |
| "rewards/length_reward": 0.0669642984867096, | |
| "rewards/similarity_reward": 0.5841279327869415, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 306.2857208251953, | |
| "epoch": 0.04533333333333334, | |
| "grad_norm": 0.6883394230623637, | |
| "kl": 0.00019121170043945312, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0685, | |
| "reward": 0.5360667556524277, | |
| "reward_std": 0.17403440922498703, | |
| "rewards/length_reward": 0.04866072162985802, | |
| "rewards/similarity_reward": 0.48740604519844055, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 332.41966247558594, | |
| "epoch": 0.048, | |
| "grad_norm": 0.5705875740256017, | |
| "kl": 0.00019073486328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0755, | |
| "reward": 0.6520929932594299, | |
| "reward_std": 0.11369618400931358, | |
| "rewards/length_reward": 0.06428572162985802, | |
| "rewards/similarity_reward": 0.5878072679042816, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 285.03126525878906, | |
| "epoch": 0.050666666666666665, | |
| "grad_norm": 0.6938225330516384, | |
| "kl": 0.00016689300537109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1144, | |
| "reward": 0.6287901103496552, | |
| "reward_std": 0.14219095557928085, | |
| "rewards/length_reward": 0.06071429327130318, | |
| "rewards/similarity_reward": 0.5680757761001587, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 271.70982360839844, | |
| "epoch": 0.05333333333333334, | |
| "grad_norm": 0.7297528927947673, | |
| "kl": 0.00022459030151367188, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0185, | |
| "reward": 0.6177776157855988, | |
| "reward_std": 0.16364814341068268, | |
| "rewards/length_reward": 0.058482155203819275, | |
| "rewards/similarity_reward": 0.5592954754829407, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 288.85716247558594, | |
| "epoch": 0.056, | |
| "grad_norm": 0.7214386901249736, | |
| "kl": 0.0002155303955078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0434, | |
| "reward": 0.597628653049469, | |
| "reward_std": 0.14737623929977417, | |
| "rewards/length_reward": 0.06205358728766441, | |
| "rewards/similarity_reward": 0.5355750769376755, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 286.6875228881836, | |
| "epoch": 0.058666666666666666, | |
| "grad_norm": 0.6632688424489647, | |
| "kl": 0.0001938343048095703, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0844, | |
| "reward": 0.6639349460601807, | |
| "reward_std": 0.13361597061157227, | |
| "rewards/length_reward": 0.06205357797443867, | |
| "rewards/similarity_reward": 0.6018813252449036, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 274.4419708251953, | |
| "epoch": 0.06133333333333333, | |
| "grad_norm": 0.6481364691610889, | |
| "kl": 0.00017881393432617188, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0025, | |
| "reward": 0.6752828657627106, | |
| "reward_std": 0.1561322584748268, | |
| "rewards/length_reward": 0.0625000149011612, | |
| "rewards/similarity_reward": 0.612782746553421, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 286.56697845458984, | |
| "epoch": 0.064, | |
| "grad_norm": 0.6864769985687215, | |
| "kl": 0.00017595291137695312, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0716, | |
| "reward": 0.5987082123756409, | |
| "reward_std": 0.1384410411119461, | |
| "rewards/length_reward": 0.06428572349250317, | |
| "rewards/similarity_reward": 0.5344224572181702, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 279.7053680419922, | |
| "epoch": 0.06666666666666667, | |
| "grad_norm": 0.6989530874534239, | |
| "kl": 0.0001468658447265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1417, | |
| "reward": 0.7139560580253601, | |
| "reward_std": 0.12727026268839836, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.6447596251964569, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 269.00000762939453, | |
| "epoch": 0.06933333333333333, | |
| "grad_norm": 0.8072710062956403, | |
| "kl": 0.00020742416381835938, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1312, | |
| "reward": 0.5711838901042938, | |
| "reward_std": 0.16566643118858337, | |
| "rewards/length_reward": 0.055803582072257996, | |
| "rewards/similarity_reward": 0.515380322933197, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 261.1339340209961, | |
| "epoch": 0.072, | |
| "grad_norm": 0.6813535460039397, | |
| "kl": 0.00016641616821289062, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0717, | |
| "reward": 0.6621068716049194, | |
| "reward_std": 0.126276895403862, | |
| "rewards/length_reward": 0.0669642984867096, | |
| "rewards/similarity_reward": 0.5951426327228546, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 301.8169708251953, | |
| "epoch": 0.07466666666666667, | |
| "grad_norm": 0.7408476748329949, | |
| "kl": 0.00023174285888671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0763, | |
| "reward": 0.5747545063495636, | |
| "reward_std": 0.13357429951429367, | |
| "rewards/length_reward": 0.05178572237491608, | |
| "rewards/similarity_reward": 0.5229687094688416, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 292.96876525878906, | |
| "epoch": 0.07733333333333334, | |
| "grad_norm": 0.6666274319697456, | |
| "kl": 0.0001964569091796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1043, | |
| "reward": 0.6386013031005859, | |
| "reward_std": 0.12155063822865486, | |
| "rewards/length_reward": 0.0669642984867096, | |
| "rewards/similarity_reward": 0.5716370046138763, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 295.7589416503906, | |
| "epoch": 0.08, | |
| "grad_norm": 0.6572218858197006, | |
| "kl": 0.0002288818359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0662, | |
| "reward": 0.6807651817798615, | |
| "reward_std": 0.16092021018266678, | |
| "rewards/length_reward": 0.0691964328289032, | |
| "rewards/similarity_reward": 0.6115686893463135, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 307.6383972167969, | |
| "epoch": 0.08266666666666667, | |
| "grad_norm": 0.7925535842055806, | |
| "kl": 0.00022029876708984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.2064, | |
| "reward": 0.5656463354825974, | |
| "reward_std": 0.1584705486893654, | |
| "rewards/length_reward": 0.051785726100206375, | |
| "rewards/similarity_reward": 0.5138606131076813, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 232.94197845458984, | |
| "epoch": 0.08533333333333333, | |
| "grad_norm": 0.7532498555139676, | |
| "kl": 0.00029754638671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0905, | |
| "reward": 0.6083326637744904, | |
| "reward_std": 0.14686080068349838, | |
| "rewards/length_reward": 0.06339287012815475, | |
| "rewards/similarity_reward": 0.544939786195755, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 282.5089416503906, | |
| "epoch": 0.088, | |
| "grad_norm": 0.8072212411776664, | |
| "kl": 0.0002989768981933594, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0575, | |
| "reward": 0.6441718935966492, | |
| "reward_std": 0.14405860379338264, | |
| "rewards/length_reward": 0.06919643096625805, | |
| "rewards/similarity_reward": 0.574975460767746, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 255.03125762939453, | |
| "epoch": 0.09066666666666667, | |
| "grad_norm": 0.7214716728436158, | |
| "kl": 0.00023508071899414062, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0549, | |
| "reward": 0.65805384516716, | |
| "reward_std": 0.10812044516205788, | |
| "rewards/length_reward": 0.054017869755625725, | |
| "rewards/similarity_reward": 0.6040360331535339, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 318.4464416503906, | |
| "epoch": 0.09333333333333334, | |
| "grad_norm": 0.6443834325802673, | |
| "kl": 0.0002117156982421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0879, | |
| "reward": 0.6600002646446228, | |
| "reward_std": 0.13175412267446518, | |
| "rewards/length_reward": 0.0656250137835741, | |
| "rewards/similarity_reward": 0.5943752527236938, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 265.6607360839844, | |
| "epoch": 0.096, | |
| "grad_norm": 0.7350551326118709, | |
| "kl": 0.000255584716796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0763, | |
| "reward": 0.6379535496234894, | |
| "reward_std": 0.11777842044830322, | |
| "rewards/length_reward": 0.061160726472735405, | |
| "rewards/similarity_reward": 0.57679283618927, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 257.3928756713867, | |
| "epoch": 0.09866666666666667, | |
| "grad_norm": 0.7838614111079478, | |
| "kl": 0.0003414154052734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0207, | |
| "reward": 0.6382779181003571, | |
| "reward_std": 0.10474509000778198, | |
| "rewards/length_reward": 0.07053572684526443, | |
| "rewards/similarity_reward": 0.567742183804512, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 281.02232360839844, | |
| "epoch": 0.10133333333333333, | |
| "grad_norm": 0.718502539431113, | |
| "kl": 0.000362396240234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0634, | |
| "reward": 0.684599369764328, | |
| "reward_std": 0.12997651100158691, | |
| "rewards/length_reward": 0.0669642947614193, | |
| "rewards/similarity_reward": 0.6176350712776184, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 263.25447845458984, | |
| "epoch": 0.104, | |
| "grad_norm": 0.6560992159018658, | |
| "kl": 0.00020933151245117188, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0628, | |
| "reward": 0.6398553550243378, | |
| "reward_std": 0.14855975657701492, | |
| "rewards/length_reward": 0.058035727590322495, | |
| "rewards/similarity_reward": 0.5818196535110474, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 315.5982360839844, | |
| "epoch": 0.10666666666666667, | |
| "grad_norm": 0.6477813051659901, | |
| "kl": 0.000324249267578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1156, | |
| "reward": 0.57817542552948, | |
| "reward_std": 0.13820311054587364, | |
| "rewards/length_reward": 0.052678581327199936, | |
| "rewards/similarity_reward": 0.525496780872345, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 239.44644165039062, | |
| "epoch": 0.10933333333333334, | |
| "grad_norm": 0.6691411219790073, | |
| "kl": 0.0003085136413574219, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0558, | |
| "reward": 0.6137453615665436, | |
| "reward_std": 0.13270848989486694, | |
| "rewards/length_reward": 0.061607152223587036, | |
| "rewards/similarity_reward": 0.5521382689476013, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 256.9866256713867, | |
| "epoch": 0.112, | |
| "grad_norm": 0.7883991158904521, | |
| "kl": 0.00034618377685546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0302, | |
| "reward": 0.5703277885913849, | |
| "reward_std": 0.16237758100032806, | |
| "rewards/length_reward": 0.04910715110599995, | |
| "rewards/similarity_reward": 0.5212206840515137, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 294.6339416503906, | |
| "epoch": 0.11466666666666667, | |
| "grad_norm": 0.6538334220808693, | |
| "kl": 0.00020074844360351562, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1178, | |
| "reward": 0.6531505286693573, | |
| "reward_std": 0.1386118158698082, | |
| "rewards/length_reward": 0.06964286789298058, | |
| "rewards/similarity_reward": 0.583507627248764, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 259.49554443359375, | |
| "epoch": 0.11733333333333333, | |
| "grad_norm": 0.6454158665655293, | |
| "kl": 0.0002779960632324219, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0229, | |
| "reward": 0.6056394279003143, | |
| "reward_std": 0.1302252970635891, | |
| "rewards/length_reward": 0.07008930295705795, | |
| "rewards/similarity_reward": 0.5355501472949982, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 281.0982360839844, | |
| "epoch": 0.12, | |
| "grad_norm": 0.819280226455994, | |
| "kl": 0.00036144256591796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1387, | |
| "reward": 0.5956102013587952, | |
| "reward_std": 0.1252099834382534, | |
| "rewards/length_reward": 0.05089286528527737, | |
| "rewards/similarity_reward": 0.5447173118591309, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 259.5848388671875, | |
| "epoch": 0.12266666666666666, | |
| "grad_norm": 0.7348999369116066, | |
| "kl": 0.00026702880859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0863, | |
| "reward": 0.6511934101581573, | |
| "reward_std": 0.1133498027920723, | |
| "rewards/length_reward": 0.0625000111758709, | |
| "rewards/similarity_reward": 0.588693380355835, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 255.55358123779297, | |
| "epoch": 0.12533333333333332, | |
| "grad_norm": 0.8374758265958845, | |
| "kl": 0.00042057037353515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.094, | |
| "reward": 0.5409015715122223, | |
| "reward_std": 0.10493671149015427, | |
| "rewards/length_reward": 0.058928582817316055, | |
| "rewards/similarity_reward": 0.48197296261787415, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 262.93304443359375, | |
| "epoch": 0.128, | |
| "grad_norm": 0.6793546139834532, | |
| "kl": 0.000423431396484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0649, | |
| "reward": 0.5910505056381226, | |
| "reward_std": 0.16320697963237762, | |
| "rewards/length_reward": 0.06428572721779346, | |
| "rewards/similarity_reward": 0.5267648100852966, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 274.53572845458984, | |
| "epoch": 0.13066666666666665, | |
| "grad_norm": 0.7145465386525555, | |
| "kl": 0.0003376007080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0209, | |
| "reward": 0.65211421251297, | |
| "reward_std": 0.16534826159477234, | |
| "rewards/length_reward": 0.06339287012815475, | |
| "rewards/similarity_reward": 0.588721364736557, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 267.5714340209961, | |
| "epoch": 0.13333333333333333, | |
| "grad_norm": 0.6146172464604014, | |
| "kl": 0.00035190582275390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0452, | |
| "reward": 0.6328105330467224, | |
| "reward_std": 0.16772306710481644, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.5662927031517029, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 244.4910888671875, | |
| "epoch": 0.136, | |
| "grad_norm": 0.9373778001138064, | |
| "kl": 0.00045299530029296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1341, | |
| "reward": 0.5600460767745972, | |
| "reward_std": 0.1334986537694931, | |
| "rewards/length_reward": 0.059375012293457985, | |
| "rewards/similarity_reward": 0.5006710439920425, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 251.4553680419922, | |
| "epoch": 0.13866666666666666, | |
| "grad_norm": 0.6828077611773323, | |
| "kl": 0.00024318695068359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0511, | |
| "reward": 0.6347568333148956, | |
| "reward_std": 0.07518525794148445, | |
| "rewards/length_reward": 0.060267867520451546, | |
| "rewards/similarity_reward": 0.5744889676570892, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 258.74554443359375, | |
| "epoch": 0.14133333333333334, | |
| "grad_norm": 0.7291338070723046, | |
| "kl": 0.00031948089599609375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0178, | |
| "reward": 0.6101614236831665, | |
| "reward_std": 0.15437154471874237, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.5436435341835022, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 275.62501525878906, | |
| "epoch": 0.144, | |
| "grad_norm": 0.5749401912567113, | |
| "kl": 0.0002770423889160156, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1447, | |
| "reward": 0.7116933763027191, | |
| "reward_std": 0.10556156933307648, | |
| "rewards/length_reward": 0.06964286789298058, | |
| "rewards/similarity_reward": 0.6420504748821259, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 297.88841247558594, | |
| "epoch": 0.14666666666666667, | |
| "grad_norm": 0.6712391841955945, | |
| "kl": 0.00043582916259765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0543, | |
| "reward": 0.6665964424610138, | |
| "reward_std": 0.13994581252336502, | |
| "rewards/length_reward": 0.07232143357396126, | |
| "rewards/similarity_reward": 0.5942749679088593, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 261.2946548461914, | |
| "epoch": 0.14933333333333335, | |
| "grad_norm": 0.7543785996872888, | |
| "kl": 0.000606536865234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0927, | |
| "reward": 0.5725409537553787, | |
| "reward_std": 0.19671519845724106, | |
| "rewards/length_reward": 0.05178572237491608, | |
| "rewards/similarity_reward": 0.5207552313804626, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 232.9196548461914, | |
| "epoch": 0.152, | |
| "grad_norm": 0.6310665650295014, | |
| "kl": 0.00034427642822265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0251, | |
| "reward": 0.710765928030014, | |
| "reward_std": 0.11330895498394966, | |
| "rewards/length_reward": 0.07678571715950966, | |
| "rewards/similarity_reward": 0.6339801549911499, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 275.8839340209961, | |
| "epoch": 0.15466666666666667, | |
| "grad_norm": 0.6081646973355043, | |
| "kl": 0.00048542022705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0153, | |
| "reward": 0.634895533323288, | |
| "reward_std": 0.12484097108244896, | |
| "rewards/length_reward": 0.0736607201397419, | |
| "rewards/similarity_reward": 0.5612348020076752, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 296.9151916503906, | |
| "epoch": 0.15733333333333333, | |
| "grad_norm": 0.6547927211710071, | |
| "kl": 0.0003204345703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.101, | |
| "reward": 0.679833859205246, | |
| "reward_std": 0.13348159193992615, | |
| "rewards/length_reward": 0.07053572311997414, | |
| "rewards/similarity_reward": 0.6092981398105621, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 281.6919708251953, | |
| "epoch": 0.16, | |
| "grad_norm": 0.6398386646259059, | |
| "kl": 0.000492095947265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0297, | |
| "reward": 0.6490921080112457, | |
| "reward_std": 0.14558688551187515, | |
| "rewards/length_reward": 0.07008930295705795, | |
| "rewards/similarity_reward": 0.5790028274059296, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 255.53126525878906, | |
| "epoch": 0.16266666666666665, | |
| "grad_norm": 0.7177382491967137, | |
| "kl": 0.0004730224609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0431, | |
| "reward": 0.6999586224555969, | |
| "reward_std": 0.13094842806458473, | |
| "rewards/length_reward": 0.06383929774165154, | |
| "rewards/similarity_reward": 0.6361193954944611, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 269.15626525878906, | |
| "epoch": 0.16533333333333333, | |
| "grad_norm": 0.5891947471632187, | |
| "kl": 0.00038623809814453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0092, | |
| "reward": 0.7129235565662384, | |
| "reward_std": 0.0941559188067913, | |
| "rewards/length_reward": 0.07455357909202576, | |
| "rewards/similarity_reward": 0.6383699178695679, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 294.62054443359375, | |
| "epoch": 0.168, | |
| "grad_norm": 0.8226717384917975, | |
| "kl": 0.0005388259887695312, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0827, | |
| "reward": 0.6617253720760345, | |
| "reward_std": 0.10956033691763878, | |
| "rewards/length_reward": 0.06250001303851604, | |
| "rewards/similarity_reward": 0.5992253720760345, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 265.2410888671875, | |
| "epoch": 0.17066666666666666, | |
| "grad_norm": 0.7642480042855918, | |
| "kl": 0.0007228851318359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0968, | |
| "reward": 0.6403190791606903, | |
| "reward_std": 0.1368250623345375, | |
| "rewards/length_reward": 0.06071429327130318, | |
| "rewards/similarity_reward": 0.5796047747135162, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 241.55358123779297, | |
| "epoch": 0.17333333333333334, | |
| "grad_norm": 0.8292179764665186, | |
| "kl": 0.00046443939208984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.065, | |
| "reward": 0.6242024898529053, | |
| "reward_std": 0.15151464939117432, | |
| "rewards/length_reward": 0.05491072125732899, | |
| "rewards/similarity_reward": 0.5692917704582214, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 234.7946548461914, | |
| "epoch": 0.176, | |
| "grad_norm": 0.6954246830187802, | |
| "kl": 0.00040721893310546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0081, | |
| "reward": 0.6392101645469666, | |
| "reward_std": 0.13006018847227097, | |
| "rewards/length_reward": 0.06562500819563866, | |
| "rewards/similarity_reward": 0.5735851526260376, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 272.9776916503906, | |
| "epoch": 0.17866666666666667, | |
| "grad_norm": 0.8379515100728363, | |
| "kl": 0.0006618499755859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0778, | |
| "reward": 0.6054639518260956, | |
| "reward_std": 0.15866044908761978, | |
| "rewards/length_reward": 0.056250011548399925, | |
| "rewards/similarity_reward": 0.5492139458656311, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 246.08482360839844, | |
| "epoch": 0.18133333333333335, | |
| "grad_norm": 0.6633622096307521, | |
| "kl": 0.000507354736328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0327, | |
| "reward": 0.6771285533905029, | |
| "reward_std": 0.11619659885764122, | |
| "rewards/length_reward": 0.06696430034935474, | |
| "rewards/similarity_reward": 0.6101642847061157, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 260.4776916503906, | |
| "epoch": 0.184, | |
| "grad_norm": 0.7742612358878467, | |
| "kl": 0.0005159378051757812, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0148, | |
| "reward": 0.711969256401062, | |
| "reward_std": 0.14254993572831154, | |
| "rewards/length_reward": 0.07276786491274834, | |
| "rewards/similarity_reward": 0.6392013728618622, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 218.6696548461914, | |
| "epoch": 0.18666666666666668, | |
| "grad_norm": 0.7878879808209311, | |
| "kl": 0.0006504058837890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1186, | |
| "reward": 0.5868320167064667, | |
| "reward_std": 0.139366053044796, | |
| "rewards/length_reward": 0.050892867147922516, | |
| "rewards/similarity_reward": 0.5359391123056412, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 273.07591247558594, | |
| "epoch": 0.18933333333333333, | |
| "grad_norm": 0.6879154147040387, | |
| "kl": 0.00072479248046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0361, | |
| "reward": 0.5751383006572723, | |
| "reward_std": 0.16684868186712265, | |
| "rewards/length_reward": 0.061607155948877335, | |
| "rewards/similarity_reward": 0.5135311335325241, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 271.7053680419922, | |
| "epoch": 0.192, | |
| "grad_norm": 0.6918784121385549, | |
| "kl": 0.0006055831909179688, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0412, | |
| "reward": 0.6015307903289795, | |
| "reward_std": 0.11276621744036674, | |
| "rewards/length_reward": 0.06383929401636124, | |
| "rewards/similarity_reward": 0.5376915037631989, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 239.18304443359375, | |
| "epoch": 0.19466666666666665, | |
| "grad_norm": 0.6763207999389166, | |
| "kl": 0.00046539306640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0378, | |
| "reward": 0.738762378692627, | |
| "reward_std": 0.07676012441515923, | |
| "rewards/length_reward": 0.0669642947614193, | |
| "rewards/similarity_reward": 0.671798050403595, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 298.2321472167969, | |
| "epoch": 0.19733333333333333, | |
| "grad_norm": 0.7183360474036766, | |
| "kl": 0.00069427490234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1536, | |
| "reward": 0.6193753182888031, | |
| "reward_std": 0.13081318512558937, | |
| "rewards/length_reward": 0.055803585797548294, | |
| "rewards/similarity_reward": 0.5635717213153839, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 278.71876525878906, | |
| "epoch": 0.2, | |
| "grad_norm": 0.6422209743450289, | |
| "kl": 0.0005578994750976562, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1141, | |
| "reward": 0.7105484306812286, | |
| "reward_std": 0.1271527223289013, | |
| "rewards/length_reward": 0.0758928656578064, | |
| "rewards/similarity_reward": 0.6346555352210999, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 232.0759048461914, | |
| "epoch": 0.20266666666666666, | |
| "grad_norm": 0.8972600571762434, | |
| "kl": 0.0006427764892578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.051, | |
| "reward": 0.5786014497280121, | |
| "reward_std": 0.13573814183473587, | |
| "rewards/length_reward": 0.06964286416769028, | |
| "rewards/similarity_reward": 0.5089586079120636, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 281.2232208251953, | |
| "epoch": 0.20533333333333334, | |
| "grad_norm": 0.8130156685147141, | |
| "kl": 0.00061798095703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1302, | |
| "reward": 0.6210056841373444, | |
| "reward_std": 0.19511007517576218, | |
| "rewards/length_reward": 0.053125010803341866, | |
| "rewards/similarity_reward": 0.5678807199001312, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 258.6651916503906, | |
| "epoch": 0.208, | |
| "grad_norm": 0.7255155232620163, | |
| "kl": 0.0008106231689453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0153, | |
| "reward": 0.619497686624527, | |
| "reward_std": 0.10088447853922844, | |
| "rewards/length_reward": 0.060267867520451546, | |
| "rewards/similarity_reward": 0.5592298209667206, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 255.79466247558594, | |
| "epoch": 0.21066666666666667, | |
| "grad_norm": 0.6729672044279214, | |
| "kl": 0.000576019287109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0819, | |
| "reward": 0.7009203135967255, | |
| "reward_std": 0.10652491822838783, | |
| "rewards/length_reward": 0.07142858020961285, | |
| "rewards/similarity_reward": 0.6294918060302734, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 275.15626525878906, | |
| "epoch": 0.21333333333333335, | |
| "grad_norm": 0.7063326351608399, | |
| "kl": 0.0008029937744140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.106, | |
| "reward": 0.6244928240776062, | |
| "reward_std": 0.14966875314712524, | |
| "rewards/length_reward": 0.056696439161896706, | |
| "rewards/similarity_reward": 0.5677963495254517, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 280.9464416503906, | |
| "epoch": 0.216, | |
| "grad_norm": 0.5928309311542868, | |
| "kl": 0.00060272216796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0107, | |
| "reward": 0.6575084030628204, | |
| "reward_std": 0.10019119456410408, | |
| "rewards/length_reward": 0.06964286789298058, | |
| "rewards/similarity_reward": 0.587865561246872, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 232.42858123779297, | |
| "epoch": 0.21866666666666668, | |
| "grad_norm": 0.728258017728098, | |
| "kl": 0.0008029937744140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0327, | |
| "reward": 0.6456558704376221, | |
| "reward_std": 0.10026798397302628, | |
| "rewards/length_reward": 0.07187500596046448, | |
| "rewards/similarity_reward": 0.5737808644771576, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 272.4241180419922, | |
| "epoch": 0.22133333333333333, | |
| "grad_norm": 0.7474863840166143, | |
| "kl": 0.0006437301635742188, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0835, | |
| "reward": 0.6149618327617645, | |
| "reward_std": 0.11428236961364746, | |
| "rewards/length_reward": 0.06205357797443867, | |
| "rewards/similarity_reward": 0.552908256649971, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 281.40179443359375, | |
| "epoch": 0.224, | |
| "grad_norm": 0.5698111928088367, | |
| "kl": 0.0008392333984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0067, | |
| "reward": 0.6963447332382202, | |
| "reward_std": 0.10215538740158081, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.6280410885810852, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 249.6696548461914, | |
| "epoch": 0.22666666666666666, | |
| "grad_norm": 0.774725228553314, | |
| "kl": 0.0007877349853515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0906, | |
| "reward": 0.6118488311767578, | |
| "reward_std": 0.14368778094649315, | |
| "rewards/length_reward": 0.06428572349250317, | |
| "rewards/similarity_reward": 0.5475630760192871, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 265.3482208251953, | |
| "epoch": 0.22933333333333333, | |
| "grad_norm": 0.8923548112445079, | |
| "kl": 0.0011768341064453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0958, | |
| "reward": 0.6147017180919647, | |
| "reward_std": 0.1361084319651127, | |
| "rewards/length_reward": 0.057589296251535416, | |
| "rewards/similarity_reward": 0.557112455368042, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 251.7901840209961, | |
| "epoch": 0.232, | |
| "grad_norm": 0.618718290820211, | |
| "kl": 0.0007457733154296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0706, | |
| "reward": 0.6785232722759247, | |
| "reward_std": 0.09245472028851509, | |
| "rewards/length_reward": 0.07500000670552254, | |
| "rewards/similarity_reward": 0.6035232245922089, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 311.21876525878906, | |
| "epoch": 0.23466666666666666, | |
| "grad_norm": 0.5515407468114694, | |
| "kl": 0.000797271728515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.012, | |
| "reward": 0.6955881416797638, | |
| "reward_std": 0.10647468641400337, | |
| "rewards/length_reward": 0.08303571492433548, | |
| "rewards/similarity_reward": 0.6125523746013641, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 241.92858123779297, | |
| "epoch": 0.23733333333333334, | |
| "grad_norm": 0.7246692380980303, | |
| "kl": 0.0008525848388671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0111, | |
| "reward": 0.7196685373783112, | |
| "reward_std": 0.11786854639649391, | |
| "rewards/length_reward": 0.06562501564621925, | |
| "rewards/similarity_reward": 0.6540435552597046, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 266.3482208251953, | |
| "epoch": 0.24, | |
| "grad_norm": 0.8337410916485611, | |
| "kl": 0.00109100341796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0753, | |
| "reward": 0.6227089762687683, | |
| "reward_std": 0.11949346587061882, | |
| "rewards/length_reward": 0.05178572237491608, | |
| "rewards/similarity_reward": 0.570923238992691, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 312.4553680419922, | |
| "epoch": 0.24266666666666667, | |
| "grad_norm": 0.7247658170183164, | |
| "kl": 0.0009250640869140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.074, | |
| "reward": 0.6411847472190857, | |
| "reward_std": 0.15169727057218552, | |
| "rewards/length_reward": 0.06651787087321281, | |
| "rewards/similarity_reward": 0.5746668875217438, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 240.4821548461914, | |
| "epoch": 0.24533333333333332, | |
| "grad_norm": 0.7784189357920823, | |
| "kl": 0.000835418701171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0335, | |
| "reward": 0.7014745771884918, | |
| "reward_std": 0.08627640455961227, | |
| "rewards/length_reward": 0.06517857871949673, | |
| "rewards/similarity_reward": 0.6362960338592529, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 253.05805206298828, | |
| "epoch": 0.248, | |
| "grad_norm": 0.7201282634879667, | |
| "kl": 0.000904083251953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0981, | |
| "reward": 0.6915363669395447, | |
| "reward_std": 0.14938431978225708, | |
| "rewards/length_reward": 0.06875001266598701, | |
| "rewards/similarity_reward": 0.6227863430976868, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 281.4776916503906, | |
| "epoch": 0.25066666666666665, | |
| "grad_norm": 0.6973337009868545, | |
| "kl": 0.0007686614990234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0789, | |
| "reward": 0.6481258869171143, | |
| "reward_std": 0.10927876830101013, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.5798223316669464, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 205.28125762939453, | |
| "epoch": 0.25333333333333335, | |
| "grad_norm": 0.8693659568274648, | |
| "kl": 0.00103759765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0431, | |
| "reward": 0.6439125239849091, | |
| "reward_std": 0.11249563843011856, | |
| "rewards/length_reward": 0.058035727590322495, | |
| "rewards/similarity_reward": 0.5858767628669739, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 292.4241180419922, | |
| "epoch": 0.256, | |
| "grad_norm": 0.5457303629614552, | |
| "kl": 0.0009708404541015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0148, | |
| "reward": 0.7119653820991516, | |
| "reward_std": 0.1094476543366909, | |
| "rewards/length_reward": 0.08169642835855484, | |
| "rewards/similarity_reward": 0.630268931388855, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 300.5089416503906, | |
| "epoch": 0.25866666666666666, | |
| "grad_norm": 0.5831645575431905, | |
| "kl": 0.0009822845458984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0146, | |
| "reward": 0.6134711802005768, | |
| "reward_std": 0.10632903501391411, | |
| "rewards/length_reward": 0.07142858020961285, | |
| "rewards/similarity_reward": 0.5420425981283188, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 239.30358123779297, | |
| "epoch": 0.2613333333333333, | |
| "grad_norm": 0.7748344420880229, | |
| "kl": 0.0009174346923828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0451, | |
| "reward": 0.682037353515625, | |
| "reward_std": 0.08752915635704994, | |
| "rewards/length_reward": 0.07410715520381927, | |
| "rewards/similarity_reward": 0.6079301834106445, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 247.00001525878906, | |
| "epoch": 0.264, | |
| "grad_norm": 0.6574205211811521, | |
| "kl": 0.00130462646484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0383, | |
| "reward": 0.6871787905693054, | |
| "reward_std": 0.12282633036375046, | |
| "rewards/length_reward": 0.06607143767178059, | |
| "rewards/similarity_reward": 0.6211073100566864, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 255.9866180419922, | |
| "epoch": 0.26666666666666666, | |
| "grad_norm": 0.6550349068969015, | |
| "kl": 0.0011653900146484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.031, | |
| "reward": 0.6847736537456512, | |
| "reward_std": 0.11585507914423943, | |
| "rewards/length_reward": 0.07767857611179352, | |
| "rewards/similarity_reward": 0.6070950329303741, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 302.05804443359375, | |
| "epoch": 0.2693333333333333, | |
| "grad_norm": 0.6581037881537074, | |
| "kl": 0.0008869171142578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0275, | |
| "reward": 0.7058568000793457, | |
| "reward_std": 0.1038212925195694, | |
| "rewards/length_reward": 0.0736607238650322, | |
| "rewards/similarity_reward": 0.6321960687637329, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 252.37055206298828, | |
| "epoch": 0.272, | |
| "grad_norm": 0.7659892153308909, | |
| "kl": 0.0009002685546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0099, | |
| "reward": 0.6600384712219238, | |
| "reward_std": 0.11620288342237473, | |
| "rewards/length_reward": 0.06964286416769028, | |
| "rewards/similarity_reward": 0.5903956294059753, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 260.9910888671875, | |
| "epoch": 0.27466666666666667, | |
| "grad_norm": 0.7873803837848522, | |
| "kl": 0.0011749267578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.067, | |
| "reward": 0.647940456867218, | |
| "reward_std": 0.12016744539141655, | |
| "rewards/length_reward": 0.060267869383096695, | |
| "rewards/similarity_reward": 0.587672621011734, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 303.0982208251953, | |
| "epoch": 0.2773333333333333, | |
| "grad_norm": 0.6698404225945914, | |
| "kl": 0.0008029937744140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0326, | |
| "reward": 0.6724097728729248, | |
| "reward_std": 0.08444990962743759, | |
| "rewards/length_reward": 0.07232143729925156, | |
| "rewards/similarity_reward": 0.600088357925415, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 263.43751525878906, | |
| "epoch": 0.28, | |
| "grad_norm": 0.7035133703979454, | |
| "kl": 0.0010471343994140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0377, | |
| "reward": 0.6536130607128143, | |
| "reward_std": 0.16632840782403946, | |
| "rewards/length_reward": 0.06741072982549667, | |
| "rewards/similarity_reward": 0.5862023532390594, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 265.3526916503906, | |
| "epoch": 0.2826666666666667, | |
| "grad_norm": 0.7008056428586112, | |
| "kl": 0.0012378692626953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0104, | |
| "reward": 0.6467622220516205, | |
| "reward_std": 0.11927535384893417, | |
| "rewards/length_reward": 0.06651787087321281, | |
| "rewards/similarity_reward": 0.5802443325519562, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 275.6026916503906, | |
| "epoch": 0.2853333333333333, | |
| "grad_norm": 0.6158480236246975, | |
| "kl": 0.001171112060546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0208, | |
| "reward": 0.6745641827583313, | |
| "reward_std": 0.11500725150108337, | |
| "rewards/length_reward": 0.06964286416769028, | |
| "rewards/similarity_reward": 0.604921281337738, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 260.33036041259766, | |
| "epoch": 0.288, | |
| "grad_norm": 0.7224564328760787, | |
| "kl": 0.000965118408203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0761, | |
| "reward": 0.5630394518375397, | |
| "reward_std": 0.1106732226908207, | |
| "rewards/length_reward": 0.053571442142128944, | |
| "rewards/similarity_reward": 0.5094679594039917, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 243.68751525878906, | |
| "epoch": 0.2906666666666667, | |
| "grad_norm": 0.8254406902750699, | |
| "kl": 0.001445770263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0425, | |
| "reward": 0.6382045447826385, | |
| "reward_std": 0.12056771293282509, | |
| "rewards/length_reward": 0.062053581699728966, | |
| "rewards/similarity_reward": 0.5761510133743286, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 224.87501525878906, | |
| "epoch": 0.29333333333333333, | |
| "grad_norm": 0.6268606551971334, | |
| "kl": 0.0011119842529296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0105, | |
| "reward": 0.5971523225307465, | |
| "reward_std": 0.10623245313763618, | |
| "rewards/length_reward": 0.06517858244478703, | |
| "rewards/similarity_reward": 0.5319737493991852, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 259.1339340209961, | |
| "epoch": 0.296, | |
| "grad_norm": 0.6687342397274337, | |
| "kl": 0.00101470947265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1205, | |
| "reward": 0.6370533108711243, | |
| "reward_std": 0.12068188562989235, | |
| "rewards/length_reward": 0.06294644251465797, | |
| "rewards/similarity_reward": 0.5741068720817566, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 249.50000762939453, | |
| "epoch": 0.2986666666666667, | |
| "grad_norm": 0.6806425164227126, | |
| "kl": 0.001422882080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.052, | |
| "reward": 0.6264582276344299, | |
| "reward_std": 0.11265718936920166, | |
| "rewards/length_reward": 0.06428572908043861, | |
| "rewards/similarity_reward": 0.562172532081604, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 270.3883972167969, | |
| "epoch": 0.30133333333333334, | |
| "grad_norm": 0.7014564695453412, | |
| "kl": 0.0009479522705078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1126, | |
| "reward": 0.7100532650947571, | |
| "reward_std": 0.10190926492214203, | |
| "rewards/length_reward": 0.07857143878936768, | |
| "rewards/similarity_reward": 0.6314818263053894, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 271.99554443359375, | |
| "epoch": 0.304, | |
| "grad_norm": 0.8072459229444039, | |
| "kl": 0.001361846923828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1128, | |
| "reward": 0.6362347304821014, | |
| "reward_std": 0.13582201302051544, | |
| "rewards/length_reward": 0.07008929550647736, | |
| "rewards/similarity_reward": 0.5661454498767853, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 223.00447845458984, | |
| "epoch": 0.30666666666666664, | |
| "grad_norm": 0.8550809137589263, | |
| "kl": 0.001277923583984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1073, | |
| "reward": 0.6763125956058502, | |
| "reward_std": 0.11202903836965561, | |
| "rewards/length_reward": 0.06607143580913544, | |
| "rewards/similarity_reward": 0.6102411448955536, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 279.54466247558594, | |
| "epoch": 0.30933333333333335, | |
| "grad_norm": 0.7634314721408118, | |
| "kl": 0.001033782958984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1038, | |
| "reward": 0.6390451192855835, | |
| "reward_std": 0.11136174201965332, | |
| "rewards/length_reward": 0.0714285746216774, | |
| "rewards/similarity_reward": 0.5676165223121643, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 251.08929443359375, | |
| "epoch": 0.312, | |
| "grad_norm": 0.8512986272863483, | |
| "kl": 0.0017547607421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.027, | |
| "reward": 0.5662170350551605, | |
| "reward_std": 0.1223873421549797, | |
| "rewards/length_reward": 0.051785726100206375, | |
| "rewards/similarity_reward": 0.5144313126802444, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 279.3526916503906, | |
| "epoch": 0.31466666666666665, | |
| "grad_norm": 0.5648142162422048, | |
| "kl": 0.001270294189453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.058, | |
| "reward": 0.6961429119110107, | |
| "reward_std": 0.08081653341650963, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.6296250522136688, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 228.95536041259766, | |
| "epoch": 0.31733333333333336, | |
| "grad_norm": 0.87246537686519, | |
| "kl": 0.00152587890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0655, | |
| "reward": 0.5725010931491852, | |
| "reward_std": 0.13679825142025948, | |
| "rewards/length_reward": 0.051785724237561226, | |
| "rewards/similarity_reward": 0.5207154005765915, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 213.12500762939453, | |
| "epoch": 0.32, | |
| "grad_norm": 0.90775103522726, | |
| "kl": 0.00186920166015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0752, | |
| "reward": 0.5904355347156525, | |
| "reward_std": 0.13604873418807983, | |
| "rewards/length_reward": 0.060714298859238625, | |
| "rewards/similarity_reward": 0.5297212451696396, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 272.4598388671875, | |
| "epoch": 0.32266666666666666, | |
| "grad_norm": 0.679207876261083, | |
| "kl": 0.001651763916015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0208, | |
| "reward": 0.5465417206287384, | |
| "reward_std": 0.11999019607901573, | |
| "rewards/length_reward": 0.060714296996593475, | |
| "rewards/similarity_reward": 0.48582740128040314, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 264.77679443359375, | |
| "epoch": 0.3253333333333333, | |
| "grad_norm": 0.6586415817875254, | |
| "kl": 0.0012359619140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0806, | |
| "reward": 0.68158820271492, | |
| "reward_std": 0.1025921143591404, | |
| "rewards/length_reward": 0.07008929923176765, | |
| "rewards/similarity_reward": 0.6114989817142487, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 256.1607208251953, | |
| "epoch": 0.328, | |
| "grad_norm": 0.7273109756128965, | |
| "kl": 0.00136566162109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0542, | |
| "reward": 0.6582387089729309, | |
| "reward_std": 0.10396385937929153, | |
| "rewards/length_reward": 0.062053581699728966, | |
| "rewards/similarity_reward": 0.5961851477622986, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 258.7723388671875, | |
| "epoch": 0.33066666666666666, | |
| "grad_norm": 0.7158563788139325, | |
| "kl": 0.0016632080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0495, | |
| "reward": 0.680817037820816, | |
| "reward_std": 0.15591518580913544, | |
| "rewards/length_reward": 0.0669643022119999, | |
| "rewards/similarity_reward": 0.6138526499271393, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 287.1026916503906, | |
| "epoch": 0.3333333333333333, | |
| "grad_norm": 0.7257840755435857, | |
| "kl": 0.0021820068359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1379, | |
| "reward": 0.6818304061889648, | |
| "reward_std": 0.14189621806144714, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.6153125464916229, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 272.3794860839844, | |
| "epoch": 0.336, | |
| "grad_norm": 0.716044862480554, | |
| "kl": 0.00128936767578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0606, | |
| "reward": 0.607832282781601, | |
| "reward_std": 0.10822133347392082, | |
| "rewards/length_reward": 0.06517858244478703, | |
| "rewards/similarity_reward": 0.5426536649465561, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 267.2008972167969, | |
| "epoch": 0.33866666666666667, | |
| "grad_norm": 0.7078650786713803, | |
| "kl": 0.0017242431640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0553, | |
| "reward": 0.6737089157104492, | |
| "reward_std": 0.12828165292739868, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.6054053008556366, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 247.7276840209961, | |
| "epoch": 0.3413333333333333, | |
| "grad_norm": 0.9316222904188678, | |
| "kl": 0.0016021728515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0704, | |
| "reward": 0.6324192881584167, | |
| "reward_std": 0.09700313210487366, | |
| "rewards/length_reward": 0.056250011548399925, | |
| "rewards/similarity_reward": 0.5761693120002747, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 266.2946548461914, | |
| "epoch": 0.344, | |
| "grad_norm": 0.6821391109445131, | |
| "kl": 0.00136566162109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0922, | |
| "reward": 0.600318193435669, | |
| "reward_std": 0.14177118614315987, | |
| "rewards/length_reward": 0.05892857722938061, | |
| "rewards/similarity_reward": 0.5413895845413208, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 260.25447845458984, | |
| "epoch": 0.3466666666666667, | |
| "grad_norm": 0.6135827945376331, | |
| "kl": 0.001003265380859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0975, | |
| "reward": 0.6941237151622772, | |
| "reward_std": 0.11082329601049423, | |
| "rewards/length_reward": 0.07232144102454185, | |
| "rewards/similarity_reward": 0.6218023300170898, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 301.4598388671875, | |
| "epoch": 0.34933333333333333, | |
| "grad_norm": 0.727973261017594, | |
| "kl": 0.00146484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1049, | |
| "reward": 0.6211672425270081, | |
| "reward_std": 0.1372002437710762, | |
| "rewards/length_reward": 0.05625000782310963, | |
| "rewards/similarity_reward": 0.5649172067642212, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 235.4910888671875, | |
| "epoch": 0.352, | |
| "grad_norm": 0.7386524195731257, | |
| "kl": 0.0012664794921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0175, | |
| "reward": 0.6908943951129913, | |
| "reward_std": 0.09667576849460602, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.6225908398628235, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 220.09376525878906, | |
| "epoch": 0.3546666666666667, | |
| "grad_norm": 0.6544388171069107, | |
| "kl": 0.00138092041015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0025, | |
| "reward": 0.6801804006099701, | |
| "reward_std": 0.11540298722684383, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.6118768155574799, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 273.5714340209961, | |
| "epoch": 0.35733333333333334, | |
| "grad_norm": 0.5808123143329609, | |
| "kl": 0.0009899139404296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0302, | |
| "reward": 0.6981923878192902, | |
| "reward_std": 0.12756088376045227, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.6267638206481934, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 273.83483123779297, | |
| "epoch": 0.36, | |
| "grad_norm": 0.6257800546867194, | |
| "kl": 0.001750946044921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0255, | |
| "reward": 0.6324395537376404, | |
| "reward_std": 0.11662106215953827, | |
| "rewards/length_reward": 0.07276786491274834, | |
| "rewards/similarity_reward": 0.5596716403961182, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 209.4821548461914, | |
| "epoch": 0.3626666666666667, | |
| "grad_norm": 0.7210088344519984, | |
| "kl": 0.00167083740234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0129, | |
| "reward": 0.6599233448505402, | |
| "reward_std": 0.08075730502605438, | |
| "rewards/length_reward": 0.06339286640286446, | |
| "rewards/similarity_reward": 0.5965304374694824, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 258.7232208251953, | |
| "epoch": 0.36533333333333334, | |
| "grad_norm": 0.7922066598101163, | |
| "kl": 0.001499176025390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0838, | |
| "reward": 0.6245545744895935, | |
| "reward_std": 0.12391168251633644, | |
| "rewards/length_reward": 0.0625000111758709, | |
| "rewards/similarity_reward": 0.5620545446872711, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 290.34822845458984, | |
| "epoch": 0.368, | |
| "grad_norm": 0.6230240946470257, | |
| "kl": 0.00208282470703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0447, | |
| "reward": 0.6634125411510468, | |
| "reward_std": 0.10979421064257622, | |
| "rewards/length_reward": 0.06428572535514832, | |
| "rewards/similarity_reward": 0.599126785993576, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 265.3839416503906, | |
| "epoch": 0.37066666666666664, | |
| "grad_norm": 0.7078887986803306, | |
| "kl": 0.002201080322265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0458, | |
| "reward": 0.70221146941185, | |
| "reward_std": 0.10299094393849373, | |
| "rewards/length_reward": 0.06964286789298058, | |
| "rewards/similarity_reward": 0.6325685381889343, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 265.7678680419922, | |
| "epoch": 0.37333333333333335, | |
| "grad_norm": 0.7842037190011205, | |
| "kl": 0.001491546630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0973, | |
| "reward": 0.644274890422821, | |
| "reward_std": 0.10034830123186111, | |
| "rewards/length_reward": 0.061160728335380554, | |
| "rewards/similarity_reward": 0.5831141769886017, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 249.66964721679688, | |
| "epoch": 0.376, | |
| "grad_norm": 0.567490680527846, | |
| "kl": 0.001689910888671875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.024, | |
| "reward": 0.7013588547706604, | |
| "reward_std": 0.0840650200843811, | |
| "rewards/length_reward": 0.06651786342263222, | |
| "rewards/similarity_reward": 0.6348409652709961, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 226.25447845458984, | |
| "epoch": 0.37866666666666665, | |
| "grad_norm": 0.912319551814626, | |
| "kl": 0.001934051513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0961, | |
| "reward": 0.6543315052986145, | |
| "reward_std": 0.10859474539756775, | |
| "rewards/length_reward": 0.06607143953442574, | |
| "rewards/similarity_reward": 0.5882599949836731, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 298.8616180419922, | |
| "epoch": 0.38133333333333336, | |
| "grad_norm": 0.5902099764974178, | |
| "kl": 0.001720428466796875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0059, | |
| "reward": 0.7150795757770538, | |
| "reward_std": 0.08074402436614037, | |
| "rewards/length_reward": 0.0781250111758709, | |
| "rewards/similarity_reward": 0.6369545459747314, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 258.6026916503906, | |
| "epoch": 0.384, | |
| "grad_norm": 0.7182388748669096, | |
| "kl": 0.0016632080078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.073, | |
| "reward": 0.6721713840961456, | |
| "reward_std": 0.10838266462087631, | |
| "rewards/length_reward": 0.07321429625153542, | |
| "rewards/similarity_reward": 0.5989570617675781, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 249.93304443359375, | |
| "epoch": 0.38666666666666666, | |
| "grad_norm": 0.736716904979314, | |
| "kl": 0.001873016357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0098, | |
| "reward": 0.6520076394081116, | |
| "reward_std": 0.08915667980909348, | |
| "rewards/length_reward": 0.059821439906954765, | |
| "rewards/similarity_reward": 0.5921862125396729, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 249.5401840209961, | |
| "epoch": 0.3893333333333333, | |
| "grad_norm": 0.696504045867252, | |
| "kl": 0.0015106201171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0287, | |
| "reward": 0.6746790409088135, | |
| "reward_std": 0.10656709223985672, | |
| "rewards/length_reward": 0.06785715371370316, | |
| "rewards/similarity_reward": 0.6068219244480133, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 335.7098388671875, | |
| "epoch": 0.392, | |
| "grad_norm": 0.5956601576776329, | |
| "kl": 0.001354217529296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0678, | |
| "reward": 0.6324039399623871, | |
| "reward_std": 0.0887768529355526, | |
| "rewards/length_reward": 0.07053572684526443, | |
| "rewards/similarity_reward": 0.5618681907653809, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 245.93304443359375, | |
| "epoch": 0.39466666666666667, | |
| "grad_norm": 0.7100692950335996, | |
| "kl": 0.001987457275390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0427, | |
| "reward": 0.6061528325080872, | |
| "reward_std": 0.09401173889636993, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.5378492623567581, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 220.54911041259766, | |
| "epoch": 0.3973333333333333, | |
| "grad_norm": 0.8672496323443106, | |
| "kl": 0.00212860107421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0339, | |
| "reward": 0.6478115916252136, | |
| "reward_std": 0.12049812451004982, | |
| "rewards/length_reward": 0.06562500819563866, | |
| "rewards/similarity_reward": 0.5821865797042847, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 277.59376525878906, | |
| "epoch": 0.4, | |
| "grad_norm": 0.808658829177956, | |
| "kl": 0.002712249755859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0533, | |
| "reward": 0.6419509947299957, | |
| "reward_std": 0.11133093386888504, | |
| "rewards/length_reward": 0.06517858430743217, | |
| "rewards/similarity_reward": 0.5767723917961121, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 284.6026916503906, | |
| "epoch": 0.4026666666666667, | |
| "grad_norm": 0.5726944777875907, | |
| "kl": 0.0017871856689453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0081, | |
| "reward": 0.6313992738723755, | |
| "reward_std": 0.1251782327890396, | |
| "rewards/length_reward": 0.07008929550647736, | |
| "rewards/similarity_reward": 0.5613099485635757, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 269.39286041259766, | |
| "epoch": 0.4053333333333333, | |
| "grad_norm": 0.6397037674423297, | |
| "kl": 0.001956939697265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0305, | |
| "reward": 0.6884091794490814, | |
| "reward_std": 0.08797129616141319, | |
| "rewards/length_reward": 0.07187500968575478, | |
| "rewards/similarity_reward": 0.6165341138839722, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 249.98214721679688, | |
| "epoch": 0.408, | |
| "grad_norm": 0.7056757365461955, | |
| "kl": 0.001522064208984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.051, | |
| "reward": 0.6953409612178802, | |
| "reward_std": 0.10321801900863647, | |
| "rewards/length_reward": 0.07321429625153542, | |
| "rewards/similarity_reward": 0.6221266388893127, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 301.4196472167969, | |
| "epoch": 0.4106666666666667, | |
| "grad_norm": 0.672677703608687, | |
| "kl": 0.001739501953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0471, | |
| "reward": 0.6628727614879608, | |
| "reward_std": 0.09228364005684853, | |
| "rewards/length_reward": 0.06339287012815475, | |
| "rewards/similarity_reward": 0.5994798243045807, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 255.06697845458984, | |
| "epoch": 0.41333333333333333, | |
| "grad_norm": 0.7128891033714289, | |
| "kl": 0.00139617919921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1005, | |
| "reward": 0.7157993614673615, | |
| "reward_std": 0.09617481008172035, | |
| "rewards/length_reward": 0.06339286454021931, | |
| "rewards/similarity_reward": 0.6524064540863037, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 283.53126525878906, | |
| "epoch": 0.416, | |
| "grad_norm": 0.6441555100001085, | |
| "kl": 0.00176239013671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0274, | |
| "reward": 0.7534910440444946, | |
| "reward_std": 0.08639111369848251, | |
| "rewards/length_reward": 0.0781250037252903, | |
| "rewards/similarity_reward": 0.675366073846817, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 238.80805206298828, | |
| "epoch": 0.4186666666666667, | |
| "grad_norm": 0.6969348755424246, | |
| "kl": 0.002040863037109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0883, | |
| "reward": 0.7300522923469543, | |
| "reward_std": 0.0814172811806202, | |
| "rewards/length_reward": 0.07991071790456772, | |
| "rewards/similarity_reward": 0.650141566991806, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 277.8526840209961, | |
| "epoch": 0.42133333333333334, | |
| "grad_norm": 0.6143618956533469, | |
| "kl": 0.002044677734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0655, | |
| "reward": 0.6692302823066711, | |
| "reward_std": 0.11947489529848099, | |
| "rewards/length_reward": 0.07053572311997414, | |
| "rewards/similarity_reward": 0.5986945033073425, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 272.3526916503906, | |
| "epoch": 0.424, | |
| "grad_norm": 0.7167795024215189, | |
| "kl": 0.00182342529296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0687, | |
| "reward": 0.6641163527965546, | |
| "reward_std": 0.11917684972286224, | |
| "rewards/length_reward": 0.0625000111758709, | |
| "rewards/similarity_reward": 0.6016163527965546, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 248.9821548461914, | |
| "epoch": 0.4266666666666667, | |
| "grad_norm": 0.7327545151839033, | |
| "kl": 0.00356292724609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0745, | |
| "reward": 0.6552646160125732, | |
| "reward_std": 0.11251871287822723, | |
| "rewards/length_reward": 0.06294644065201283, | |
| "rewards/similarity_reward": 0.5923182666301727, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 256.07591247558594, | |
| "epoch": 0.42933333333333334, | |
| "grad_norm": 0.7535276542214958, | |
| "kl": 0.0027008056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0314, | |
| "reward": 0.6351377367973328, | |
| "reward_std": 0.10685055330395699, | |
| "rewards/length_reward": 0.059375012293457985, | |
| "rewards/similarity_reward": 0.5757627189159393, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 257.43750762939453, | |
| "epoch": 0.432, | |
| "grad_norm": 0.6595271125555469, | |
| "kl": 0.00250244140625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0056, | |
| "reward": 0.6979295313358307, | |
| "reward_std": 0.09611150622367859, | |
| "rewards/length_reward": 0.07187500968575478, | |
| "rewards/similarity_reward": 0.6260544955730438, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 262.4419860839844, | |
| "epoch": 0.43466666666666665, | |
| "grad_norm": 0.6185607216367858, | |
| "kl": 0.002216339111328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0996, | |
| "reward": 0.6647298634052277, | |
| "reward_std": 0.15266689658164978, | |
| "rewards/length_reward": 0.06964286416769028, | |
| "rewards/similarity_reward": 0.5950870215892792, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 307.3526916503906, | |
| "epoch": 0.43733333333333335, | |
| "grad_norm": 0.7121773803870707, | |
| "kl": 0.001800537109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0489, | |
| "reward": 0.7000181972980499, | |
| "reward_std": 0.12559669092297554, | |
| "rewards/length_reward": 0.06741072237491608, | |
| "rewards/similarity_reward": 0.6326074600219727, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 246.79019165039062, | |
| "epoch": 0.44, | |
| "grad_norm": 0.6721019148616484, | |
| "kl": 0.00197601318359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0431, | |
| "reward": 0.7078234553337097, | |
| "reward_std": 0.09047579020261765, | |
| "rewards/length_reward": 0.06741072610020638, | |
| "rewards/similarity_reward": 0.6404128074645996, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 272.7723388671875, | |
| "epoch": 0.44266666666666665, | |
| "grad_norm": 0.5673761372545018, | |
| "kl": 0.002292633056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0246, | |
| "reward": 0.7385044991970062, | |
| "reward_std": 0.07343112863600254, | |
| "rewards/length_reward": 0.0714285783469677, | |
| "rewards/similarity_reward": 0.6670758426189423, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 236.49108123779297, | |
| "epoch": 0.44533333333333336, | |
| "grad_norm": 0.7731297375677283, | |
| "kl": 0.00337982177734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0703, | |
| "reward": 0.6287428140640259, | |
| "reward_std": 0.13660584390163422, | |
| "rewards/length_reward": 0.058928582817316055, | |
| "rewards/similarity_reward": 0.5698141753673553, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 250.63839721679688, | |
| "epoch": 0.448, | |
| "grad_norm": 0.6844249610340352, | |
| "kl": 0.002716064453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.005, | |
| "reward": 0.6882721185684204, | |
| "reward_std": 0.08635392412543297, | |
| "rewards/length_reward": 0.07276786863803864, | |
| "rewards/similarity_reward": 0.6155042350292206, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 299.9151916503906, | |
| "epoch": 0.45066666666666666, | |
| "grad_norm": 0.5305327006388908, | |
| "kl": 0.001537322998046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.009, | |
| "reward": 0.7003590762615204, | |
| "reward_std": 0.06643648073077202, | |
| "rewards/length_reward": 0.0781250074505806, | |
| "rewards/similarity_reward": 0.6222340762615204, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 274.7143096923828, | |
| "epoch": 0.4533333333333333, | |
| "grad_norm": 0.6964990599197315, | |
| "kl": 0.002025604248046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0422, | |
| "reward": 0.6924863159656525, | |
| "reward_std": 0.10944930091500282, | |
| "rewards/length_reward": 0.06830358505249023, | |
| "rewards/similarity_reward": 0.6241826713085175, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 260.29911041259766, | |
| "epoch": 0.456, | |
| "grad_norm": 0.7007252255586068, | |
| "kl": 0.0022430419921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0892, | |
| "reward": 0.6610677242279053, | |
| "reward_std": 0.08702437207102776, | |
| "rewards/length_reward": 0.0625000111758709, | |
| "rewards/similarity_reward": 0.5985676944255829, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 291.8883972167969, | |
| "epoch": 0.45866666666666667, | |
| "grad_norm": 0.6818242026608151, | |
| "kl": 0.00199127197265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0296, | |
| "reward": 0.6590520441532135, | |
| "reward_std": 0.11536738649010658, | |
| "rewards/length_reward": 0.058035723865032196, | |
| "rewards/similarity_reward": 0.6010163724422455, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 262.5982360839844, | |
| "epoch": 0.4613333333333333, | |
| "grad_norm": 0.7058845306121532, | |
| "kl": 0.00278472900390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0147, | |
| "reward": 0.6392101943492889, | |
| "reward_std": 0.112693652510643, | |
| "rewards/length_reward": 0.0736607238650322, | |
| "rewards/similarity_reward": 0.5655494332313538, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 243.60269165039062, | |
| "epoch": 0.464, | |
| "grad_norm": 0.6664009632817656, | |
| "kl": 0.00189208984375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0054, | |
| "reward": 0.6843527555465698, | |
| "reward_std": 0.1105504259467125, | |
| "rewards/length_reward": 0.06964286416769028, | |
| "rewards/similarity_reward": 0.6147099137306213, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 250.46875762939453, | |
| "epoch": 0.4666666666666667, | |
| "grad_norm": 0.6473092015687406, | |
| "kl": 0.00228118896484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0463, | |
| "reward": 0.6191678941249847, | |
| "reward_std": 0.1144786849617958, | |
| "rewards/length_reward": 0.06294644251465797, | |
| "rewards/similarity_reward": 0.5562214553356171, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 246.2276840209961, | |
| "epoch": 0.4693333333333333, | |
| "grad_norm": 0.782293454462268, | |
| "kl": 0.00267791748046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.104, | |
| "reward": 0.6215286254882812, | |
| "reward_std": 0.10633666813373566, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.552332192659378, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 288.9598388671875, | |
| "epoch": 0.472, | |
| "grad_norm": 0.6068876990848109, | |
| "kl": 0.0028076171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0499, | |
| "reward": 0.6609079539775848, | |
| "reward_std": 0.12471455708146095, | |
| "rewards/length_reward": 0.06294644251465797, | |
| "rewards/similarity_reward": 0.5979615151882172, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 231.6919708251953, | |
| "epoch": 0.4746666666666667, | |
| "grad_norm": 0.6072574366547386, | |
| "kl": 0.00218963623046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0049, | |
| "reward": 0.6936376392841339, | |
| "reward_std": 0.09063273295760155, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.6244412362575531, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 244.82144165039062, | |
| "epoch": 0.47733333333333333, | |
| "grad_norm": 0.6731140452564113, | |
| "kl": 0.002288818359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0199, | |
| "reward": 0.6732902526855469, | |
| "reward_std": 0.12003319337964058, | |
| "rewards/length_reward": 0.06741072982549667, | |
| "rewards/similarity_reward": 0.6058795154094696, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 248.89733123779297, | |
| "epoch": 0.48, | |
| "grad_norm": 0.6436090869352918, | |
| "kl": 0.00261688232421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0504, | |
| "reward": 0.6384674608707428, | |
| "reward_std": 0.13010089099407196, | |
| "rewards/length_reward": 0.057589298114180565, | |
| "rewards/similarity_reward": 0.5808781981468201, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 246.43304443359375, | |
| "epoch": 0.4826666666666667, | |
| "grad_norm": 0.6282260624355238, | |
| "kl": 0.005481719970703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0045, | |
| "reward": 0.717792272567749, | |
| "reward_std": 0.07219249755144119, | |
| "rewards/length_reward": 0.07008929550647736, | |
| "rewards/similarity_reward": 0.6477029919624329, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 250.3660888671875, | |
| "epoch": 0.48533333333333334, | |
| "grad_norm": 0.8187805221092193, | |
| "kl": 0.00237274169921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0499, | |
| "reward": 0.6446585357189178, | |
| "reward_std": 0.1213221587240696, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.573229968547821, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 238.0535888671875, | |
| "epoch": 0.488, | |
| "grad_norm": 0.7872695013078711, | |
| "kl": 0.0045166015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0325, | |
| "reward": 0.6350027918815613, | |
| "reward_std": 0.08915703371167183, | |
| "rewards/length_reward": 0.06205358728766441, | |
| "rewards/similarity_reward": 0.5729491412639618, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 239.0446548461914, | |
| "epoch": 0.49066666666666664, | |
| "grad_norm": 0.6850244703959089, | |
| "kl": 0.002655029296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0088, | |
| "reward": 0.6566590070724487, | |
| "reward_std": 0.08741006627678871, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.5901412069797516, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 228.43304443359375, | |
| "epoch": 0.49333333333333335, | |
| "grad_norm": 0.640590428870281, | |
| "kl": 0.00246429443359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0211, | |
| "reward": 0.7332558631896973, | |
| "reward_std": 0.08987564593553543, | |
| "rewards/length_reward": 0.07098215445876122, | |
| "rewards/similarity_reward": 0.6622737348079681, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 275.46875762939453, | |
| "epoch": 0.496, | |
| "grad_norm": 0.7034238309978142, | |
| "kl": 0.00243377685546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0345, | |
| "reward": 0.7175212800502777, | |
| "reward_std": 0.08901329711079597, | |
| "rewards/length_reward": 0.07544643431901932, | |
| "rewards/similarity_reward": 0.6420748233795166, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 280.00000762939453, | |
| "epoch": 0.49866666666666665, | |
| "grad_norm": 0.657474847366372, | |
| "kl": 0.00211334228515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0942, | |
| "reward": 0.702696293592453, | |
| "reward_std": 0.08004150912165642, | |
| "rewards/length_reward": 0.0669642984867096, | |
| "rewards/similarity_reward": 0.635731965303421, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 246.83929443359375, | |
| "epoch": 0.5013333333333333, | |
| "grad_norm": 0.6840180489866146, | |
| "kl": 0.002269744873046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0305, | |
| "reward": 0.6730459034442902, | |
| "reward_std": 0.08726384490728378, | |
| "rewards/length_reward": 0.07053572684526443, | |
| "rewards/similarity_reward": 0.6025101542472839, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 253.83929443359375, | |
| "epoch": 0.504, | |
| "grad_norm": 0.5974802952790658, | |
| "kl": 0.00257110595703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0762, | |
| "reward": 0.6664688587188721, | |
| "reward_std": 0.11599006876349449, | |
| "rewards/length_reward": 0.07723214477300644, | |
| "rewards/similarity_reward": 0.5892367660999298, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 278.80804443359375, | |
| "epoch": 0.5066666666666667, | |
| "grad_norm": 0.6466637537752649, | |
| "kl": 0.00311279296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0959, | |
| "reward": 0.6832419335842133, | |
| "reward_std": 0.11445752903819084, | |
| "rewards/length_reward": 0.06830358505249023, | |
| "rewards/similarity_reward": 0.614938348531723, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 235.5491180419922, | |
| "epoch": 0.5093333333333333, | |
| "grad_norm": 0.7153062964233342, | |
| "kl": 0.0019378662109375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0045, | |
| "reward": 0.6687261462211609, | |
| "reward_std": 0.09642783552408218, | |
| "rewards/length_reward": 0.06562500447034836, | |
| "rewards/similarity_reward": 0.6031011044979095, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 252.80358123779297, | |
| "epoch": 0.512, | |
| "grad_norm": 0.6313047035707975, | |
| "kl": 0.00281524658203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0303, | |
| "reward": 0.6900187730789185, | |
| "reward_std": 0.10082340613007545, | |
| "rewards/length_reward": 0.07008929923176765, | |
| "rewards/similarity_reward": 0.6199295520782471, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 297.0580596923828, | |
| "epoch": 0.5146666666666667, | |
| "grad_norm": 0.6070377598000647, | |
| "kl": 0.001934051513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0237, | |
| "reward": 0.7174617052078247, | |
| "reward_std": 0.10485106706619263, | |
| "rewards/length_reward": 0.07544643804430962, | |
| "rewards/similarity_reward": 0.642015278339386, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 235.87500762939453, | |
| "epoch": 0.5173333333333333, | |
| "grad_norm": 0.6962391024556379, | |
| "kl": 0.00244140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0154, | |
| "reward": 0.64629727602005, | |
| "reward_std": 0.0772336795926094, | |
| "rewards/length_reward": 0.07008929550647736, | |
| "rewards/similarity_reward": 0.5762079358100891, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 206.8482208251953, | |
| "epoch": 0.52, | |
| "grad_norm": 0.8558133302744291, | |
| "kl": 0.00318145751953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0683, | |
| "reward": 0.6385850310325623, | |
| "reward_std": 0.0856214202940464, | |
| "rewards/length_reward": 0.06517857871949673, | |
| "rewards/similarity_reward": 0.5734064280986786, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 306.2633972167969, | |
| "epoch": 0.5226666666666666, | |
| "grad_norm": 0.5439460695909182, | |
| "kl": 0.0030517578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0583, | |
| "reward": 0.6826609075069427, | |
| "reward_std": 0.08685707673430443, | |
| "rewards/length_reward": 0.07857143133878708, | |
| "rewards/similarity_reward": 0.6040894687175751, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 284.8169708251953, | |
| "epoch": 0.5253333333333333, | |
| "grad_norm": 0.597796232109364, | |
| "kl": 0.00238800048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0081, | |
| "reward": 0.7354407906532288, | |
| "reward_std": 0.06834723800420761, | |
| "rewards/length_reward": 0.07500000670552254, | |
| "rewards/similarity_reward": 0.6604407727718353, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 271.2901916503906, | |
| "epoch": 0.528, | |
| "grad_norm": 0.7779344829533027, | |
| "kl": 0.00433349609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1361, | |
| "reward": 0.6831106841564178, | |
| "reward_std": 0.16374623030424118, | |
| "rewards/length_reward": 0.06875000894069672, | |
| "rewards/similarity_reward": 0.6143606305122375, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 257.1294708251953, | |
| "epoch": 0.5306666666666666, | |
| "grad_norm": 0.6115200963053203, | |
| "kl": 0.002285003662109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0286, | |
| "reward": 0.6817789673805237, | |
| "reward_std": 0.07882888615131378, | |
| "rewards/length_reward": 0.07857143133878708, | |
| "rewards/similarity_reward": 0.6032074391841888, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 232.77679443359375, | |
| "epoch": 0.5333333333333333, | |
| "grad_norm": 0.7171528103086203, | |
| "kl": 0.0023651123046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0367, | |
| "reward": 0.7090138792991638, | |
| "reward_std": 0.1117062047123909, | |
| "rewards/length_reward": 0.06741072610020638, | |
| "rewards/similarity_reward": 0.6416031718254089, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 292.0446472167969, | |
| "epoch": 0.536, | |
| "grad_norm": 0.6795728974282024, | |
| "kl": 0.0023193359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0438, | |
| "reward": 0.6276125609874725, | |
| "reward_std": 0.09431122243404388, | |
| "rewards/length_reward": 0.06875000894069672, | |
| "rewards/similarity_reward": 0.5588625073432922, | |
| "step": 201 | |
| }, | |
| { | |
| "completion_length": 211.61161041259766, | |
| "epoch": 0.5386666666666666, | |
| "grad_norm": 0.9251828325921423, | |
| "kl": 0.0043487548828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0439, | |
| "reward": 0.5786231458187103, | |
| "reward_std": 0.1044333316385746, | |
| "rewards/length_reward": 0.058928582817316055, | |
| "rewards/similarity_reward": 0.5196945518255234, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 241.27680206298828, | |
| "epoch": 0.5413333333333333, | |
| "grad_norm": 0.6521091947128457, | |
| "kl": 0.00319671630859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0361, | |
| "reward": 0.6666609346866608, | |
| "reward_std": 0.092064518481493, | |
| "rewards/length_reward": 0.07544643431901932, | |
| "rewards/similarity_reward": 0.5912144482135773, | |
| "step": 203 | |
| }, | |
| { | |
| "completion_length": 211.28125762939453, | |
| "epoch": 0.544, | |
| "grad_norm": 0.6743614543760093, | |
| "kl": 0.00234222412109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0135, | |
| "reward": 0.6388188004493713, | |
| "reward_std": 0.11344650015234947, | |
| "rewards/length_reward": 0.06785715371370316, | |
| "rewards/similarity_reward": 0.570961594581604, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 279.90179443359375, | |
| "epoch": 0.5466666666666666, | |
| "grad_norm": 0.6332153076352647, | |
| "kl": 0.00287628173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0423, | |
| "reward": 0.655907928943634, | |
| "reward_std": 0.0848810151219368, | |
| "rewards/length_reward": 0.07187500968575478, | |
| "rewards/similarity_reward": 0.5840329229831696, | |
| "step": 205 | |
| }, | |
| { | |
| "completion_length": 201.8169708251953, | |
| "epoch": 0.5493333333333333, | |
| "grad_norm": 0.85099814384088, | |
| "kl": 0.00351715087890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0037, | |
| "reward": 0.5818662643432617, | |
| "reward_std": 0.09155267104506493, | |
| "rewards/length_reward": 0.058482155203819275, | |
| "rewards/similarity_reward": 0.5233840942382812, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 265.27679443359375, | |
| "epoch": 0.552, | |
| "grad_norm": 0.6261335014996906, | |
| "kl": 0.00209808349609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0181, | |
| "reward": 0.7043133974075317, | |
| "reward_std": 0.09633800387382507, | |
| "rewards/length_reward": 0.06875001266598701, | |
| "rewards/similarity_reward": 0.6355634033679962, | |
| "step": 207 | |
| }, | |
| { | |
| "completion_length": 280.0669860839844, | |
| "epoch": 0.5546666666666666, | |
| "grad_norm": 0.6804923498664105, | |
| "kl": 0.00217437744140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0146, | |
| "reward": 0.6569553017616272, | |
| "reward_std": 0.08773703873157501, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.585526704788208, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 274.3482208251953, | |
| "epoch": 0.5573333333333333, | |
| "grad_norm": 0.7277401805105727, | |
| "kl": 0.0023345947265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1005, | |
| "reward": 0.7013810873031616, | |
| "reward_std": 0.09931684285402298, | |
| "rewards/length_reward": 0.07455357909202576, | |
| "rewards/similarity_reward": 0.6268274784088135, | |
| "step": 209 | |
| }, | |
| { | |
| "completion_length": 260.08036041259766, | |
| "epoch": 0.56, | |
| "grad_norm": 0.7807615068386718, | |
| "kl": 0.002655029296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0773, | |
| "reward": 0.6310366690158844, | |
| "reward_std": 0.10352174565196037, | |
| "rewards/length_reward": 0.062053579837083817, | |
| "rewards/similarity_reward": 0.5689830929040909, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 259.75447845458984, | |
| "epoch": 0.5626666666666666, | |
| "grad_norm": 0.7093685270251532, | |
| "kl": 0.0039825439453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0714, | |
| "reward": 0.6279307305812836, | |
| "reward_std": 0.10738151893019676, | |
| "rewards/length_reward": 0.06607143767178059, | |
| "rewards/similarity_reward": 0.5618593096733093, | |
| "step": 211 | |
| }, | |
| { | |
| "completion_length": 219.70983123779297, | |
| "epoch": 0.5653333333333334, | |
| "grad_norm": 0.8108428594239084, | |
| "kl": 0.00278472900390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0942, | |
| "reward": 0.6396916508674622, | |
| "reward_std": 0.10166310518980026, | |
| "rewards/length_reward": 0.06785715371370316, | |
| "rewards/similarity_reward": 0.5718345046043396, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 232.99555206298828, | |
| "epoch": 0.568, | |
| "grad_norm": 0.6621005503481664, | |
| "kl": 0.00279998779296875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0055, | |
| "reward": 0.6306145489215851, | |
| "reward_std": 0.06187950074672699, | |
| "rewards/length_reward": 0.058482153341174126, | |
| "rewards/similarity_reward": 0.5721323788166046, | |
| "step": 213 | |
| }, | |
| { | |
| "completion_length": 252.12947845458984, | |
| "epoch": 0.5706666666666667, | |
| "grad_norm": 0.5674408374580422, | |
| "kl": 0.00339508056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0245, | |
| "reward": 0.6192521750926971, | |
| "reward_std": 0.07048613205552101, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.5527342855930328, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 255.9732208251953, | |
| "epoch": 0.5733333333333334, | |
| "grad_norm": 0.6904920555987271, | |
| "kl": 0.00270843505859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0648, | |
| "reward": 0.6510066092014313, | |
| "reward_std": 0.056849006563425064, | |
| "rewards/length_reward": 0.0669642947614193, | |
| "rewards/similarity_reward": 0.5840422213077545, | |
| "step": 215 | |
| }, | |
| { | |
| "completion_length": 259.75001525878906, | |
| "epoch": 0.576, | |
| "grad_norm": 0.7131778652424658, | |
| "kl": 0.00354766845703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0514, | |
| "reward": 0.6677349805831909, | |
| "reward_std": 0.10058147087693214, | |
| "rewards/length_reward": 0.058035725727677345, | |
| "rewards/similarity_reward": 0.6096992194652557, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 259.1294708251953, | |
| "epoch": 0.5786666666666667, | |
| "grad_norm": 0.5695386420901992, | |
| "kl": 0.00286102294921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0224, | |
| "reward": 0.6971063017845154, | |
| "reward_std": 0.09981782734394073, | |
| "rewards/length_reward": 0.06741072610020638, | |
| "rewards/similarity_reward": 0.6296955347061157, | |
| "step": 217 | |
| }, | |
| { | |
| "completion_length": 236.69644927978516, | |
| "epoch": 0.5813333333333334, | |
| "grad_norm": 0.8176974415818429, | |
| "kl": 0.00765228271484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0594, | |
| "reward": 0.611341118812561, | |
| "reward_std": 0.10692798718810081, | |
| "rewards/length_reward": 0.06428572721779346, | |
| "rewards/similarity_reward": 0.5470553934574127, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 251.71875762939453, | |
| "epoch": 0.584, | |
| "grad_norm": 0.5850098610329177, | |
| "kl": 0.002689361572265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0162, | |
| "reward": 0.7100738286972046, | |
| "reward_std": 0.08857924118638039, | |
| "rewards/length_reward": 0.0714285783469677, | |
| "rewards/similarity_reward": 0.638645201921463, | |
| "step": 219 | |
| }, | |
| { | |
| "completion_length": 263.6875, | |
| "epoch": 0.5866666666666667, | |
| "grad_norm": 0.5616868949162326, | |
| "kl": 0.00293731689453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.014, | |
| "reward": 0.7153567671775818, | |
| "reward_std": 0.1150759868323803, | |
| "rewards/length_reward": 0.07857143506407738, | |
| "rewards/similarity_reward": 0.6367852687835693, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 259.24108123779297, | |
| "epoch": 0.5893333333333334, | |
| "grad_norm": 0.6551918580682736, | |
| "kl": 0.0029296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.015, | |
| "reward": 0.734772652387619, | |
| "reward_std": 0.07534275390207767, | |
| "rewards/length_reward": 0.07410714775323868, | |
| "rewards/similarity_reward": 0.6606654524803162, | |
| "step": 221 | |
| }, | |
| { | |
| "completion_length": 258.8303680419922, | |
| "epoch": 0.592, | |
| "grad_norm": 0.7150710475065644, | |
| "kl": 0.00344085693359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0202, | |
| "reward": 0.6679186522960663, | |
| "reward_std": 0.10778569802641869, | |
| "rewards/length_reward": 0.06875000521540642, | |
| "rewards/similarity_reward": 0.5991686284542084, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 263.20091247558594, | |
| "epoch": 0.5946666666666667, | |
| "grad_norm": 0.6293755467193611, | |
| "kl": 0.00225067138671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0133, | |
| "reward": 0.7472398281097412, | |
| "reward_std": 0.09251785278320312, | |
| "rewards/length_reward": 0.07723214849829674, | |
| "rewards/similarity_reward": 0.6700076460838318, | |
| "step": 223 | |
| }, | |
| { | |
| "completion_length": 270.21876525878906, | |
| "epoch": 0.5973333333333334, | |
| "grad_norm": 0.7206087593599326, | |
| "kl": 0.00225067138671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1129, | |
| "reward": 0.6448234021663666, | |
| "reward_std": 0.13600966706871986, | |
| "rewards/length_reward": 0.060714298859238625, | |
| "rewards/similarity_reward": 0.5841090679168701, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 277.3839416503906, | |
| "epoch": 0.6, | |
| "grad_norm": 0.7555717124296486, | |
| "kl": 0.003326416015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1094, | |
| "reward": 0.6888637244701385, | |
| "reward_std": 0.12441981956362724, | |
| "rewards/length_reward": 0.0669642947614193, | |
| "rewards/similarity_reward": 0.6218994557857513, | |
| "step": 225 | |
| }, | |
| { | |
| "completion_length": 288.4553680419922, | |
| "epoch": 0.6026666666666667, | |
| "grad_norm": 0.545561898466039, | |
| "kl": 0.00273895263671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0106, | |
| "reward": 0.6741566956043243, | |
| "reward_std": 0.08976901695132256, | |
| "rewards/length_reward": 0.0825892873108387, | |
| "rewards/similarity_reward": 0.5915673971176147, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 291.31251525878906, | |
| "epoch": 0.6053333333333333, | |
| "grad_norm": 0.6862146524208239, | |
| "kl": 0.00433349609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0199, | |
| "reward": 0.7247207760810852, | |
| "reward_std": 0.10038316994905472, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.6555242836475372, | |
| "step": 227 | |
| }, | |
| { | |
| "completion_length": 237.46429443359375, | |
| "epoch": 0.608, | |
| "grad_norm": 0.6514945534975937, | |
| "kl": 0.00385284423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0598, | |
| "reward": 0.639589786529541, | |
| "reward_std": 0.08173046633601189, | |
| "rewards/length_reward": 0.06517858058214188, | |
| "rewards/similarity_reward": 0.5744112133979797, | |
| "step": 228 | |
| }, | |
| { | |
| "completion_length": 258.8393020629883, | |
| "epoch": 0.6106666666666667, | |
| "grad_norm": 0.5587705250798694, | |
| "kl": 0.001895904541015625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0066, | |
| "reward": 0.7333488166332245, | |
| "reward_std": 0.07437730580568314, | |
| "rewards/length_reward": 0.07946429029107094, | |
| "rewards/similarity_reward": 0.6538845300674438, | |
| "step": 229 | |
| }, | |
| { | |
| "completion_length": 237.80358123779297, | |
| "epoch": 0.6133333333333333, | |
| "grad_norm": 0.7749006841963979, | |
| "kl": 0.00295257568359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0066, | |
| "reward": 0.6269674301147461, | |
| "reward_std": 0.09678621962666512, | |
| "rewards/length_reward": 0.06562500819563866, | |
| "rewards/similarity_reward": 0.5613424628973007, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 234.6116180419922, | |
| "epoch": 0.616, | |
| "grad_norm": 0.8857992900076056, | |
| "kl": 0.002620697021484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.036, | |
| "reward": 0.6997538506984711, | |
| "reward_std": 0.08686410635709763, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.6305573880672455, | |
| "step": 231 | |
| }, | |
| { | |
| "completion_length": 287.6339416503906, | |
| "epoch": 0.6186666666666667, | |
| "grad_norm": 0.7389133589246519, | |
| "kl": 0.00316619873046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0551, | |
| "reward": 0.6583640575408936, | |
| "reward_std": 0.1155795156955719, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.5918461382389069, | |
| "step": 232 | |
| }, | |
| { | |
| "completion_length": 288.3482208251953, | |
| "epoch": 0.6213333333333333, | |
| "grad_norm": 0.5841788620661817, | |
| "kl": 0.003204345703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0042, | |
| "reward": 0.6696169376373291, | |
| "reward_std": 0.09103860706090927, | |
| "rewards/length_reward": 0.07410714775323868, | |
| "rewards/similarity_reward": 0.5955097377300262, | |
| "step": 233 | |
| }, | |
| { | |
| "completion_length": 277.43304443359375, | |
| "epoch": 0.624, | |
| "grad_norm": 0.5987965468099127, | |
| "kl": 0.00360107421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0078, | |
| "reward": 0.7439869344234467, | |
| "reward_std": 0.08124737069010735, | |
| "rewards/length_reward": 0.07991072162985802, | |
| "rewards/similarity_reward": 0.664076179265976, | |
| "step": 234 | |
| }, | |
| { | |
| "completion_length": 264.87501525878906, | |
| "epoch": 0.6266666666666667, | |
| "grad_norm": 0.6848960422881881, | |
| "kl": 0.002899169921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.044, | |
| "reward": 0.7305464148521423, | |
| "reward_std": 0.05242483504116535, | |
| "rewards/length_reward": 0.0691964440047741, | |
| "rewards/similarity_reward": 0.6613499820232391, | |
| "step": 235 | |
| }, | |
| { | |
| "completion_length": 282.09376525878906, | |
| "epoch": 0.6293333333333333, | |
| "grad_norm": 0.6580896863492091, | |
| "kl": 0.0034637451171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0489, | |
| "reward": 0.671272337436676, | |
| "reward_std": 0.10606055706739426, | |
| "rewards/length_reward": 0.059375010430812836, | |
| "rewards/similarity_reward": 0.6118973195552826, | |
| "step": 236 | |
| }, | |
| { | |
| "completion_length": 255.2946548461914, | |
| "epoch": 0.632, | |
| "grad_norm": 0.6382622236431467, | |
| "kl": 0.00327301025390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0185, | |
| "reward": 0.7024641931056976, | |
| "reward_std": 0.09816553816199303, | |
| "rewards/length_reward": 0.07187501341104507, | |
| "rewards/similarity_reward": 0.6305891871452332, | |
| "step": 237 | |
| }, | |
| { | |
| "completion_length": 290.4375, | |
| "epoch": 0.6346666666666667, | |
| "grad_norm": 0.6729725697653197, | |
| "kl": 0.00388336181640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0427, | |
| "reward": 0.6770188510417938, | |
| "reward_std": 0.08811097219586372, | |
| "rewards/length_reward": 0.07232144102454185, | |
| "rewards/similarity_reward": 0.6046973466873169, | |
| "step": 238 | |
| }, | |
| { | |
| "completion_length": 232.1384048461914, | |
| "epoch": 0.6373333333333333, | |
| "grad_norm": 0.729361913476319, | |
| "kl": 0.0040283203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0698, | |
| "reward": 0.6408596038818359, | |
| "reward_std": 0.10725216567516327, | |
| "rewards/length_reward": 0.06741072237491608, | |
| "rewards/similarity_reward": 0.5734489262104034, | |
| "step": 239 | |
| }, | |
| { | |
| "completion_length": 284.1964416503906, | |
| "epoch": 0.64, | |
| "grad_norm": 0.6823808496445949, | |
| "kl": 0.0027923583984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0463, | |
| "reward": 0.6450429260730743, | |
| "reward_std": 0.10368061438202858, | |
| "rewards/length_reward": 0.0669642984867096, | |
| "rewards/similarity_reward": 0.57807856798172, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 260.84822845458984, | |
| "epoch": 0.6426666666666667, | |
| "grad_norm": 0.5701250379634731, | |
| "kl": 0.00244140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.025, | |
| "reward": 0.7130960524082184, | |
| "reward_std": 0.0966276079416275, | |
| "rewards/length_reward": 0.08125000074505806, | |
| "rewards/similarity_reward": 0.631846010684967, | |
| "step": 241 | |
| }, | |
| { | |
| "completion_length": 224.5669708251953, | |
| "epoch": 0.6453333333333333, | |
| "grad_norm": 0.8310327870734144, | |
| "kl": 0.0076446533203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0555, | |
| "reward": 0.5563714802265167, | |
| "reward_std": 0.12328523397445679, | |
| "rewards/length_reward": 0.054464295506477356, | |
| "rewards/similarity_reward": 0.5019071996212006, | |
| "step": 242 | |
| }, | |
| { | |
| "completion_length": 248.47322845458984, | |
| "epoch": 0.648, | |
| "grad_norm": 0.8542415499139471, | |
| "kl": 0.00330352783203125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1058, | |
| "reward": 0.6085337102413177, | |
| "reward_std": 0.12796913087368011, | |
| "rewards/length_reward": 0.056696439161896706, | |
| "rewards/similarity_reward": 0.5518373250961304, | |
| "step": 243 | |
| }, | |
| { | |
| "completion_length": 230.2321548461914, | |
| "epoch": 0.6506666666666666, | |
| "grad_norm": 0.8214724780628195, | |
| "kl": 0.0033111572265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0369, | |
| "reward": 0.6595875322818756, | |
| "reward_std": 0.07628211192786694, | |
| "rewards/length_reward": 0.0656250100582838, | |
| "rewards/similarity_reward": 0.5939625054597855, | |
| "step": 244 | |
| }, | |
| { | |
| "completion_length": 271.4419860839844, | |
| "epoch": 0.6533333333333333, | |
| "grad_norm": 0.5339131510223507, | |
| "kl": 0.0025787353515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0016, | |
| "reward": 0.7837041914463043, | |
| "reward_std": 0.07424943707883358, | |
| "rewards/length_reward": 0.07857143506407738, | |
| "rewards/similarity_reward": 0.7051327228546143, | |
| "step": 245 | |
| }, | |
| { | |
| "completion_length": 211.97322845458984, | |
| "epoch": 0.656, | |
| "grad_norm": 0.7281589894604906, | |
| "kl": 0.00310516357421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0412, | |
| "reward": 0.6736421883106232, | |
| "reward_std": 0.08142560347914696, | |
| "rewards/length_reward": 0.07455357909202576, | |
| "rewards/similarity_reward": 0.5990886092185974, | |
| "step": 246 | |
| }, | |
| { | |
| "completion_length": 244.30805206298828, | |
| "epoch": 0.6586666666666666, | |
| "grad_norm": 0.6881733301939125, | |
| "kl": 0.002948760986328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0521, | |
| "reward": 0.7186852991580963, | |
| "reward_std": 0.07804353907704353, | |
| "rewards/length_reward": 0.07544643431901932, | |
| "rewards/similarity_reward": 0.6432388722896576, | |
| "step": 247 | |
| }, | |
| { | |
| "completion_length": 204.83929443359375, | |
| "epoch": 0.6613333333333333, | |
| "grad_norm": 0.776497059513588, | |
| "kl": 0.00371551513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0183, | |
| "reward": 0.7021584212779999, | |
| "reward_std": 0.10252076759934425, | |
| "rewards/length_reward": 0.06562500819563866, | |
| "rewards/similarity_reward": 0.6365334391593933, | |
| "step": 248 | |
| }, | |
| { | |
| "completion_length": 248.63839721679688, | |
| "epoch": 0.664, | |
| "grad_norm": 0.6263280793364865, | |
| "kl": 0.0031890869140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0264, | |
| "reward": 0.7189670205116272, | |
| "reward_std": 0.07904865220189095, | |
| "rewards/length_reward": 0.07410715520381927, | |
| "rewards/similarity_reward": 0.6448598206043243, | |
| "step": 249 | |
| }, | |
| { | |
| "completion_length": 247.37501525878906, | |
| "epoch": 0.6666666666666666, | |
| "grad_norm": 0.7055287610856288, | |
| "kl": 0.0048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0909, | |
| "reward": 0.6185382604598999, | |
| "reward_std": 0.10694902017712593, | |
| "rewards/length_reward": 0.06517858430743217, | |
| "rewards/similarity_reward": 0.553359717130661, | |
| "step": 250 | |
| }, | |
| { | |
| "completion_length": 247.83482360839844, | |
| "epoch": 0.6693333333333333, | |
| "grad_norm": 0.6610447004218395, | |
| "kl": 0.0035247802734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0112, | |
| "reward": 0.5839682221412659, | |
| "reward_std": 0.0852314792573452, | |
| "rewards/length_reward": 0.0647321529686451, | |
| "rewards/similarity_reward": 0.5192360877990723, | |
| "step": 251 | |
| }, | |
| { | |
| "completion_length": 247.12501525878906, | |
| "epoch": 0.672, | |
| "grad_norm": 0.6773638528799676, | |
| "kl": 0.00295257568359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0296, | |
| "reward": 0.6655312776565552, | |
| "reward_std": 0.10929644852876663, | |
| "rewards/length_reward": 0.060267869383096695, | |
| "rewards/similarity_reward": 0.6052633821964264, | |
| "step": 252 | |
| }, | |
| { | |
| "completion_length": 221.02679443359375, | |
| "epoch": 0.6746666666666666, | |
| "grad_norm": 0.7655042728512782, | |
| "kl": 0.00439453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0034, | |
| "reward": 0.6889495253562927, | |
| "reward_std": 0.10270040482282639, | |
| "rewards/length_reward": 0.07410715147852898, | |
| "rewards/similarity_reward": 0.6148423850536346, | |
| "step": 253 | |
| }, | |
| { | |
| "completion_length": 251.55804443359375, | |
| "epoch": 0.6773333333333333, | |
| "grad_norm": 0.6277014120461456, | |
| "kl": 0.00347900390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0195, | |
| "reward": 0.6920295655727386, | |
| "reward_std": 0.07308394275605679, | |
| "rewards/length_reward": 0.07276786491274834, | |
| "rewards/similarity_reward": 0.6192616820335388, | |
| "step": 254 | |
| }, | |
| { | |
| "completion_length": 247.24108123779297, | |
| "epoch": 0.68, | |
| "grad_norm": 0.8278631058587137, | |
| "kl": 0.0052032470703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0801, | |
| "reward": 0.5797223746776581, | |
| "reward_std": 0.14392751455307007, | |
| "rewards/length_reward": 0.05446429364383221, | |
| "rewards/similarity_reward": 0.5252580344676971, | |
| "step": 255 | |
| }, | |
| { | |
| "completion_length": 283.0178680419922, | |
| "epoch": 0.6826666666666666, | |
| "grad_norm": 0.6453549089492197, | |
| "kl": 0.002655029296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0299, | |
| "reward": 0.7264284193515778, | |
| "reward_std": 0.07789020985364914, | |
| "rewards/length_reward": 0.06964286416769028, | |
| "rewards/similarity_reward": 0.6567855477333069, | |
| "step": 256 | |
| }, | |
| { | |
| "completion_length": 299.46875, | |
| "epoch": 0.6853333333333333, | |
| "grad_norm": 0.5750252920299316, | |
| "kl": 0.00243377685546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0429, | |
| "reward": 0.7198934257030487, | |
| "reward_std": 0.06834491342306137, | |
| "rewards/length_reward": 0.07232143729925156, | |
| "rewards/similarity_reward": 0.6475720107555389, | |
| "step": 257 | |
| }, | |
| { | |
| "completion_length": 250.46429443359375, | |
| "epoch": 0.688, | |
| "grad_norm": 0.6989181650264392, | |
| "kl": 0.00299835205078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0029, | |
| "reward": 0.6974795460700989, | |
| "reward_std": 0.08845745399594307, | |
| "rewards/length_reward": 0.0691964365541935, | |
| "rewards/similarity_reward": 0.6282830834388733, | |
| "step": 258 | |
| }, | |
| { | |
| "completion_length": 249.0848388671875, | |
| "epoch": 0.6906666666666667, | |
| "grad_norm": 0.768500388572952, | |
| "kl": 0.0030670166015625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.052, | |
| "reward": 0.6331368982791901, | |
| "reward_std": 0.09397462010383606, | |
| "rewards/length_reward": 0.06741072237491608, | |
| "rewards/similarity_reward": 0.5657261312007904, | |
| "step": 259 | |
| }, | |
| { | |
| "completion_length": 205.52233123779297, | |
| "epoch": 0.6933333333333334, | |
| "grad_norm": 0.8190032734025428, | |
| "kl": 0.0068511962890625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0181, | |
| "reward": 0.6275625824928284, | |
| "reward_std": 0.0882430449128151, | |
| "rewards/length_reward": 0.062053583562374115, | |
| "rewards/similarity_reward": 0.5655089616775513, | |
| "step": 260 | |
| }, | |
| { | |
| "completion_length": 227.0134048461914, | |
| "epoch": 0.696, | |
| "grad_norm": 0.6819806218771027, | |
| "kl": 0.0030364990234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0396, | |
| "reward": 0.7068421840667725, | |
| "reward_std": 0.0854947492480278, | |
| "rewards/length_reward": 0.06741072610020638, | |
| "rewards/similarity_reward": 0.63943150639534, | |
| "step": 261 | |
| }, | |
| { | |
| "completion_length": 277.65625762939453, | |
| "epoch": 0.6986666666666667, | |
| "grad_norm": 0.6041036479677462, | |
| "kl": 0.002208709716796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0162, | |
| "reward": 0.7707177698612213, | |
| "reward_std": 0.060075582936406136, | |
| "rewards/length_reward": 0.08169642835855484, | |
| "rewards/similarity_reward": 0.6890212595462799, | |
| "step": 262 | |
| }, | |
| { | |
| "completion_length": 273.18750762939453, | |
| "epoch": 0.7013333333333334, | |
| "grad_norm": 0.6716948511043099, | |
| "kl": 0.00411224365234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0791, | |
| "reward": 0.6893300712108612, | |
| "reward_std": 0.08914920315146446, | |
| "rewards/length_reward": 0.07098215073347092, | |
| "rewards/similarity_reward": 0.6183479726314545, | |
| "step": 263 | |
| }, | |
| { | |
| "completion_length": 195.03125762939453, | |
| "epoch": 0.704, | |
| "grad_norm": 0.7288219165266566, | |
| "kl": 0.0046539306640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0365, | |
| "reward": 0.6311258971691132, | |
| "reward_std": 0.08954550698399544, | |
| "rewards/length_reward": 0.06741072610020638, | |
| "rewards/similarity_reward": 0.5637151300907135, | |
| "step": 264 | |
| }, | |
| { | |
| "completion_length": 230.5714340209961, | |
| "epoch": 0.7066666666666667, | |
| "grad_norm": 0.6590047515085815, | |
| "kl": 0.00440216064453125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0116, | |
| "reward": 0.6623952388763428, | |
| "reward_std": 0.07462666183710098, | |
| "rewards/length_reward": 0.06562501192092896, | |
| "rewards/similarity_reward": 0.5967701971530914, | |
| "step": 265 | |
| }, | |
| { | |
| "completion_length": 227.8616180419922, | |
| "epoch": 0.7093333333333334, | |
| "grad_norm": 0.6922227485667228, | |
| "kl": 0.00439453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0636, | |
| "reward": 0.6782627999782562, | |
| "reward_std": 0.11519866064190865, | |
| "rewards/length_reward": 0.061607154086232185, | |
| "rewards/similarity_reward": 0.6166556179523468, | |
| "step": 266 | |
| }, | |
| { | |
| "completion_length": 257.0714416503906, | |
| "epoch": 0.712, | |
| "grad_norm": 0.5520470941775752, | |
| "kl": 0.00386810302734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0104, | |
| "reward": 0.7550633549690247, | |
| "reward_std": 0.09709116071462631, | |
| "rewards/length_reward": 0.07008930295705795, | |
| "rewards/similarity_reward": 0.6849740147590637, | |
| "step": 267 | |
| }, | |
| { | |
| "completion_length": 232.32591247558594, | |
| "epoch": 0.7146666666666667, | |
| "grad_norm": 0.7138805826191377, | |
| "kl": 0.003936767578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0331, | |
| "reward": 0.669314980506897, | |
| "reward_std": 0.09859981015324593, | |
| "rewards/length_reward": 0.06517858244478703, | |
| "rewards/similarity_reward": 0.6041364073753357, | |
| "step": 268 | |
| }, | |
| { | |
| "completion_length": 237.11162567138672, | |
| "epoch": 0.7173333333333334, | |
| "grad_norm": 0.6150700165348351, | |
| "kl": 0.00328826904296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0028, | |
| "reward": 0.7752477526664734, | |
| "reward_std": 0.057502467185258865, | |
| "rewards/length_reward": 0.0781250037252903, | |
| "rewards/similarity_reward": 0.6971226334571838, | |
| "step": 269 | |
| }, | |
| { | |
| "completion_length": 254.75447845458984, | |
| "epoch": 0.72, | |
| "grad_norm": 0.7962475762237703, | |
| "kl": 0.00536346435546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0608, | |
| "reward": 0.6644669771194458, | |
| "reward_std": 0.10142809525132179, | |
| "rewards/length_reward": 0.056250009685754776, | |
| "rewards/similarity_reward": 0.6082169711589813, | |
| "step": 270 | |
| }, | |
| { | |
| "completion_length": 263.02679443359375, | |
| "epoch": 0.7226666666666667, | |
| "grad_norm": 0.6290318550051345, | |
| "kl": 0.00321197509765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0428, | |
| "reward": 0.6874348521232605, | |
| "reward_std": 0.08607123605906963, | |
| "rewards/length_reward": 0.0691964328289032, | |
| "rewards/similarity_reward": 0.6182384490966797, | |
| "step": 271 | |
| }, | |
| { | |
| "completion_length": 253.59375762939453, | |
| "epoch": 0.7253333333333334, | |
| "grad_norm": 0.5631283015221202, | |
| "kl": 0.0034027099609375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0079, | |
| "reward": 0.7192228138446808, | |
| "reward_std": 0.05196613632142544, | |
| "rewards/length_reward": 0.08392857387661934, | |
| "rewards/similarity_reward": 0.6352941989898682, | |
| "step": 272 | |
| }, | |
| { | |
| "completion_length": 275.14732360839844, | |
| "epoch": 0.728, | |
| "grad_norm": 0.5719219206376979, | |
| "kl": 0.00231170654296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0147, | |
| "reward": 0.7571655213832855, | |
| "reward_std": 0.07733196392655373, | |
| "rewards/length_reward": 0.07678572088479996, | |
| "rewards/similarity_reward": 0.6803797781467438, | |
| "step": 273 | |
| }, | |
| { | |
| "completion_length": 235.5982208251953, | |
| "epoch": 0.7306666666666667, | |
| "grad_norm": 0.694430050013244, | |
| "kl": 0.0037841796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0622, | |
| "reward": 0.7187261283397675, | |
| "reward_std": 0.11743078380823135, | |
| "rewards/length_reward": 0.0669642947614193, | |
| "rewards/similarity_reward": 0.6517618000507355, | |
| "step": 274 | |
| }, | |
| { | |
| "completion_length": 284.7678680419922, | |
| "epoch": 0.7333333333333333, | |
| "grad_norm": 0.5786627728822946, | |
| "kl": 0.00365447998046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0085, | |
| "reward": 0.7249119877815247, | |
| "reward_std": 0.06474133767187595, | |
| "rewards/length_reward": 0.07633929327130318, | |
| "rewards/similarity_reward": 0.648572713136673, | |
| "step": 275 | |
| }, | |
| { | |
| "completion_length": 268.0580596923828, | |
| "epoch": 0.736, | |
| "grad_norm": 0.554118778302666, | |
| "kl": 0.0027008056640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0133, | |
| "reward": 0.7421092092990875, | |
| "reward_std": 0.06751231849193573, | |
| "rewards/length_reward": 0.07544643431901932, | |
| "rewards/similarity_reward": 0.6666627824306488, | |
| "step": 276 | |
| }, | |
| { | |
| "completion_length": 191.9866180419922, | |
| "epoch": 0.7386666666666667, | |
| "grad_norm": 0.7484110778385685, | |
| "kl": 0.0081329345703125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0117, | |
| "reward": 0.5663988292217255, | |
| "reward_std": 0.11341952532529831, | |
| "rewards/length_reward": 0.055357156321406364, | |
| "rewards/similarity_reward": 0.5110416412353516, | |
| "step": 277 | |
| }, | |
| { | |
| "completion_length": 262.1428680419922, | |
| "epoch": 0.7413333333333333, | |
| "grad_norm": 0.6308341820211528, | |
| "kl": 0.003448486328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0113, | |
| "reward": 0.6712258756160736, | |
| "reward_std": 0.10108800232410431, | |
| "rewards/length_reward": 0.0691964440047741, | |
| "rewards/similarity_reward": 0.6020293831825256, | |
| "step": 278 | |
| }, | |
| { | |
| "completion_length": 257.31251525878906, | |
| "epoch": 0.744, | |
| "grad_norm": 0.7444571736292193, | |
| "kl": 0.0075836181640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0194, | |
| "reward": 0.6420714855194092, | |
| "reward_std": 0.10531632602214813, | |
| "rewards/length_reward": 0.06830358505249023, | |
| "rewards/similarity_reward": 0.5737679302692413, | |
| "step": 279 | |
| }, | |
| { | |
| "completion_length": 273.0714416503906, | |
| "epoch": 0.7466666666666667, | |
| "grad_norm": 0.6772210722840282, | |
| "kl": 0.0051116943359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0147, | |
| "reward": 0.6341734230518341, | |
| "reward_std": 0.0923330970108509, | |
| "rewards/length_reward": 0.07321428880095482, | |
| "rewards/similarity_reward": 0.5609591007232666, | |
| "step": 280 | |
| }, | |
| { | |
| "completion_length": 270.4151840209961, | |
| "epoch": 0.7493333333333333, | |
| "grad_norm": 0.6993063816092533, | |
| "kl": 0.0049591064453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0501, | |
| "reward": 0.6939212083816528, | |
| "reward_std": 0.10334672406315804, | |
| "rewards/length_reward": 0.0736607201397419, | |
| "rewards/similarity_reward": 0.6202605366706848, | |
| "step": 281 | |
| }, | |
| { | |
| "completion_length": 233.01339721679688, | |
| "epoch": 0.752, | |
| "grad_norm": 0.7285449434821649, | |
| "kl": 0.0037841796875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.013, | |
| "reward": 0.6847032904624939, | |
| "reward_std": 0.06913377530872822, | |
| "rewards/length_reward": 0.06651786714792252, | |
| "rewards/similarity_reward": 0.618185430765152, | |
| "step": 282 | |
| }, | |
| { | |
| "completion_length": 238.3482208251953, | |
| "epoch": 0.7546666666666667, | |
| "grad_norm": 0.90960398931583, | |
| "kl": 0.0053253173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.019, | |
| "reward": 0.5968413352966309, | |
| "reward_std": 0.12051042169332504, | |
| "rewards/length_reward": 0.058035727590322495, | |
| "rewards/similarity_reward": 0.5388056635856628, | |
| "step": 283 | |
| }, | |
| { | |
| "completion_length": 296.4821472167969, | |
| "epoch": 0.7573333333333333, | |
| "grad_norm": 0.6037765863353864, | |
| "kl": 0.004730224609375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0322, | |
| "reward": 0.6755504310131073, | |
| "reward_std": 0.06930392794311047, | |
| "rewards/length_reward": 0.07053572684526443, | |
| "rewards/similarity_reward": 0.6050147414207458, | |
| "step": 284 | |
| }, | |
| { | |
| "completion_length": 235.1339340209961, | |
| "epoch": 0.76, | |
| "grad_norm": 0.5442699124141502, | |
| "kl": 0.002349853515625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0114, | |
| "reward": 0.7083780765533447, | |
| "reward_std": 0.06875683926045895, | |
| "rewards/length_reward": 0.07633928954601288, | |
| "rewards/similarity_reward": 0.632038801908493, | |
| "step": 285 | |
| }, | |
| { | |
| "completion_length": 259.0401916503906, | |
| "epoch": 0.7626666666666667, | |
| "grad_norm": 0.7121048521613264, | |
| "kl": 0.00444793701171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0064, | |
| "reward": 0.6680812537670135, | |
| "reward_std": 0.077706228941679, | |
| "rewards/length_reward": 0.06607143953442574, | |
| "rewards/similarity_reward": 0.6020097434520721, | |
| "step": 286 | |
| }, | |
| { | |
| "completion_length": 270.7366256713867, | |
| "epoch": 0.7653333333333333, | |
| "grad_norm": 0.6208584555138446, | |
| "kl": 0.003662109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.015, | |
| "reward": 0.706127256155014, | |
| "reward_std": 0.11445373296737671, | |
| "rewards/length_reward": 0.07053572684526443, | |
| "rewards/similarity_reward": 0.635591596364975, | |
| "step": 287 | |
| }, | |
| { | |
| "completion_length": 250.53573608398438, | |
| "epoch": 0.768, | |
| "grad_norm": 0.7385667089013174, | |
| "kl": 0.0044708251953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.032, | |
| "reward": 0.6525153517723083, | |
| "reward_std": 0.08994012698531151, | |
| "rewards/length_reward": 0.07053572311997414, | |
| "rewards/similarity_reward": 0.5819795727729797, | |
| "step": 288 | |
| }, | |
| { | |
| "completion_length": 271.4241180419922, | |
| "epoch": 0.7706666666666667, | |
| "grad_norm": 0.571768569159425, | |
| "kl": 0.00421142578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0514, | |
| "reward": 0.6859732568264008, | |
| "reward_std": 0.10971596091985703, | |
| "rewards/length_reward": 0.07410715147852898, | |
| "rewards/similarity_reward": 0.6118661463260651, | |
| "step": 289 | |
| }, | |
| { | |
| "completion_length": 279.0089416503906, | |
| "epoch": 0.7733333333333333, | |
| "grad_norm": 0.5580184118619138, | |
| "kl": 0.00385284423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0172, | |
| "reward": 0.7247184813022614, | |
| "reward_std": 0.11090399697422981, | |
| "rewards/length_reward": 0.07232144102454185, | |
| "rewards/similarity_reward": 0.652397096157074, | |
| "step": 290 | |
| }, | |
| { | |
| "completion_length": 268.5357208251953, | |
| "epoch": 0.776, | |
| "grad_norm": 0.6568855951756748, | |
| "kl": 0.0045928955078125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0124, | |
| "reward": 0.6895658373832703, | |
| "reward_std": 0.09936371073126793, | |
| "rewards/length_reward": 0.07321429252624512, | |
| "rewards/similarity_reward": 0.6163516044616699, | |
| "step": 291 | |
| }, | |
| { | |
| "completion_length": 290.41072845458984, | |
| "epoch": 0.7786666666666666, | |
| "grad_norm": 0.5426211050313083, | |
| "kl": 0.0032196044921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.019, | |
| "reward": 0.7200377881526947, | |
| "reward_std": 0.06104239821434021, | |
| "rewards/length_reward": 0.07187500968575478, | |
| "rewards/similarity_reward": 0.6481626629829407, | |
| "step": 292 | |
| }, | |
| { | |
| "completion_length": 246.53572845458984, | |
| "epoch": 0.7813333333333333, | |
| "grad_norm": 0.7741354483614902, | |
| "kl": 0.0047454833984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.05, | |
| "reward": 0.6441958248615265, | |
| "reward_std": 0.13777245581150055, | |
| "rewards/length_reward": 0.06428572908043861, | |
| "rewards/similarity_reward": 0.5799101293087006, | |
| "step": 293 | |
| }, | |
| { | |
| "completion_length": 282.1339416503906, | |
| "epoch": 0.784, | |
| "grad_norm": 0.6078928600918315, | |
| "kl": 0.0030059814453125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0654, | |
| "reward": 0.7353447079658508, | |
| "reward_std": 0.11103275418281555, | |
| "rewards/length_reward": 0.0736607238650322, | |
| "rewards/similarity_reward": 0.6616839170455933, | |
| "step": 294 | |
| }, | |
| { | |
| "completion_length": 277.2723388671875, | |
| "epoch": 0.7866666666666666, | |
| "grad_norm": 0.6681511715620417, | |
| "kl": 0.0036773681640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0328, | |
| "reward": 0.6674771308898926, | |
| "reward_std": 0.0996098667383194, | |
| "rewards/length_reward": 0.07321429252624512, | |
| "rewards/similarity_reward": 0.5942628383636475, | |
| "step": 295 | |
| }, | |
| { | |
| "completion_length": 239.18304443359375, | |
| "epoch": 0.7893333333333333, | |
| "grad_norm": 0.587894018606086, | |
| "kl": 0.0035858154296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0416, | |
| "reward": 0.6579557359218597, | |
| "reward_std": 0.08152700588107109, | |
| "rewards/length_reward": 0.06919643469154835, | |
| "rewards/similarity_reward": 0.5887593328952789, | |
| "step": 296 | |
| }, | |
| { | |
| "completion_length": 234.94197845458984, | |
| "epoch": 0.792, | |
| "grad_norm": 0.6099426936791735, | |
| "kl": 0.0045318603515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0133, | |
| "reward": 0.6847122013568878, | |
| "reward_std": 0.1407642886042595, | |
| "rewards/length_reward": 0.06830357573926449, | |
| "rewards/similarity_reward": 0.6164086163043976, | |
| "step": 297 | |
| }, | |
| { | |
| "completion_length": 238.58483123779297, | |
| "epoch": 0.7946666666666666, | |
| "grad_norm": 0.7568584099378661, | |
| "kl": 0.0032196044921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0816, | |
| "reward": 0.7135740518569946, | |
| "reward_std": 0.08400523662567139, | |
| "rewards/length_reward": 0.07678572088479996, | |
| "rewards/similarity_reward": 0.63678839802742, | |
| "step": 298 | |
| }, | |
| { | |
| "completion_length": 265.5178680419922, | |
| "epoch": 0.7973333333333333, | |
| "grad_norm": 0.559632299831719, | |
| "kl": 0.00363922119140625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0075, | |
| "reward": 0.7292493581771851, | |
| "reward_std": 0.06625411659479141, | |
| "rewards/length_reward": 0.07544643431901932, | |
| "rewards/similarity_reward": 0.6538029313087463, | |
| "step": 299 | |
| }, | |
| { | |
| "completion_length": 266.7366256713867, | |
| "epoch": 0.8, | |
| "grad_norm": 0.6245003701307471, | |
| "kl": 0.00376129150390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0309, | |
| "reward": 0.6834622323513031, | |
| "reward_std": 0.09874312952160835, | |
| "rewards/length_reward": 0.07053572684526443, | |
| "rewards/similarity_reward": 0.6129264831542969, | |
| "step": 300 | |
| }, | |
| { | |
| "completion_length": 268.4464416503906, | |
| "epoch": 0.8026666666666666, | |
| "grad_norm": 0.6001886392680107, | |
| "kl": 0.00341796875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0133, | |
| "reward": 0.7123071253299713, | |
| "reward_std": 0.05348840728402138, | |
| "rewards/length_reward": 0.06741072982549667, | |
| "rewards/similarity_reward": 0.6448963582515717, | |
| "step": 301 | |
| }, | |
| { | |
| "completion_length": 226.2321548461914, | |
| "epoch": 0.8053333333333333, | |
| "grad_norm": 0.6336093254745343, | |
| "kl": 0.00481414794921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.011, | |
| "reward": 0.6465486586093903, | |
| "reward_std": 0.09404471330344677, | |
| "rewards/length_reward": 0.06607143767178059, | |
| "rewards/similarity_reward": 0.5804772675037384, | |
| "step": 302 | |
| }, | |
| { | |
| "completion_length": 269.90179443359375, | |
| "epoch": 0.808, | |
| "grad_norm": 0.59634105150443, | |
| "kl": 0.00426483154296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0036, | |
| "reward": 0.6335479617118835, | |
| "reward_std": 0.0826010163873434, | |
| "rewards/length_reward": 0.07500001415610313, | |
| "rewards/similarity_reward": 0.5585480332374573, | |
| "step": 303 | |
| }, | |
| { | |
| "completion_length": 277.3571472167969, | |
| "epoch": 0.8106666666666666, | |
| "grad_norm": 0.6002262528664385, | |
| "kl": 0.00386810302734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.081, | |
| "reward": 0.6462258398532867, | |
| "reward_std": 0.10935888066887856, | |
| "rewards/length_reward": 0.06875000894069672, | |
| "rewards/similarity_reward": 0.5774758458137512, | |
| "step": 304 | |
| }, | |
| { | |
| "completion_length": 291.7276916503906, | |
| "epoch": 0.8133333333333334, | |
| "grad_norm": 0.6664322809544898, | |
| "kl": 0.0032958984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0743, | |
| "reward": 0.754040002822876, | |
| "reward_std": 0.07836959138512611, | |
| "rewards/length_reward": 0.07455357536673546, | |
| "rewards/similarity_reward": 0.6794863939285278, | |
| "step": 305 | |
| }, | |
| { | |
| "completion_length": 256.62501525878906, | |
| "epoch": 0.816, | |
| "grad_norm": 0.6918122711031781, | |
| "kl": 0.003997802734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0019, | |
| "reward": 0.651542603969574, | |
| "reward_std": 0.11015984788537025, | |
| "rewards/length_reward": 0.06383929774165154, | |
| "rewards/similarity_reward": 0.587703287601471, | |
| "step": 306 | |
| }, | |
| { | |
| "completion_length": 269.5803680419922, | |
| "epoch": 0.8186666666666667, | |
| "grad_norm": 0.6835757950482498, | |
| "kl": 0.0058746337890625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0137, | |
| "reward": 0.6834548711776733, | |
| "reward_std": 0.09235357493162155, | |
| "rewards/length_reward": 0.06517858058214188, | |
| "rewards/similarity_reward": 0.6182762682437897, | |
| "step": 307 | |
| }, | |
| { | |
| "completion_length": 249.43751525878906, | |
| "epoch": 0.8213333333333334, | |
| "grad_norm": 0.6594580830657123, | |
| "kl": 0.00449371337890625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0019, | |
| "reward": 0.6838014125823975, | |
| "reward_std": 0.12604531273245811, | |
| "rewards/length_reward": 0.07008929550647736, | |
| "rewards/similarity_reward": 0.6137120425701141, | |
| "step": 308 | |
| }, | |
| { | |
| "completion_length": 272.4151916503906, | |
| "epoch": 0.824, | |
| "grad_norm": 0.616617412578935, | |
| "kl": 0.00392913818359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0372, | |
| "reward": 0.7449055016040802, | |
| "reward_std": 0.08290699683129787, | |
| "rewards/length_reward": 0.07276786491274834, | |
| "rewards/similarity_reward": 0.6721376478672028, | |
| "step": 309 | |
| }, | |
| { | |
| "completion_length": 255.8660888671875, | |
| "epoch": 0.8266666666666667, | |
| "grad_norm": 0.6324884031752415, | |
| "kl": 0.00347900390625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.014, | |
| "reward": 0.6722660958766937, | |
| "reward_std": 0.08029109984636307, | |
| "rewards/length_reward": 0.07232143729925156, | |
| "rewards/similarity_reward": 0.5999446511268616, | |
| "step": 310 | |
| }, | |
| { | |
| "completion_length": 229.05358123779297, | |
| "epoch": 0.8293333333333334, | |
| "grad_norm": 0.8069840909052901, | |
| "kl": 0.00518798828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0471, | |
| "reward": 0.6889167726039886, | |
| "reward_std": 0.10098548233509064, | |
| "rewards/length_reward": 0.0714285783469677, | |
| "rewards/similarity_reward": 0.6174881756305695, | |
| "step": 311 | |
| }, | |
| { | |
| "completion_length": 244.28125762939453, | |
| "epoch": 0.832, | |
| "grad_norm": 0.6703576381079311, | |
| "kl": 0.00594329833984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0115, | |
| "reward": 0.7172508239746094, | |
| "reward_std": 0.07987817749381065, | |
| "rewards/length_reward": 0.07008929178118706, | |
| "rewards/similarity_reward": 0.647161453962326, | |
| "step": 312 | |
| }, | |
| { | |
| "completion_length": 235.50000762939453, | |
| "epoch": 0.8346666666666667, | |
| "grad_norm": 0.6595218129039281, | |
| "kl": 0.00487518310546875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0028, | |
| "reward": 0.661961704492569, | |
| "reward_std": 0.09113451465964317, | |
| "rewards/length_reward": 0.06607143767178059, | |
| "rewards/similarity_reward": 0.5958903431892395, | |
| "step": 313 | |
| }, | |
| { | |
| "completion_length": 219.86608123779297, | |
| "epoch": 0.8373333333333334, | |
| "grad_norm": 0.7020826871224611, | |
| "kl": 0.0062713623046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0162, | |
| "reward": 0.6874544024467468, | |
| "reward_std": 0.07596220076084137, | |
| "rewards/length_reward": 0.0647321566939354, | |
| "rewards/similarity_reward": 0.6227222084999084, | |
| "step": 314 | |
| }, | |
| { | |
| "completion_length": 264.8616180419922, | |
| "epoch": 0.84, | |
| "grad_norm": 0.586765871425098, | |
| "kl": 0.00507354736328125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0005, | |
| "reward": 0.7921656966209412, | |
| "reward_std": 0.0773557759821415, | |
| "rewards/length_reward": 0.0781250074505806, | |
| "rewards/similarity_reward": 0.7140407264232635, | |
| "step": 315 | |
| }, | |
| { | |
| "completion_length": 268.8348388671875, | |
| "epoch": 0.8426666666666667, | |
| "grad_norm": 0.556140838025419, | |
| "kl": 0.004425048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0045, | |
| "reward": 0.7406611740589142, | |
| "reward_std": 0.07832159847021103, | |
| "rewards/length_reward": 0.07232143729925156, | |
| "rewards/similarity_reward": 0.668339729309082, | |
| "step": 316 | |
| }, | |
| { | |
| "completion_length": 286.6696548461914, | |
| "epoch": 0.8453333333333334, | |
| "grad_norm": 0.6317684457245647, | |
| "kl": 0.00397491455078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0177, | |
| "reward": 0.791156679391861, | |
| "reward_std": 0.06108024716377258, | |
| "rewards/length_reward": 0.0781250074505806, | |
| "rewards/similarity_reward": 0.7130315899848938, | |
| "step": 317 | |
| }, | |
| { | |
| "completion_length": 223.43305206298828, | |
| "epoch": 0.848, | |
| "grad_norm": 0.7967746076439991, | |
| "kl": 0.0052642822265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0238, | |
| "reward": 0.7096598744392395, | |
| "reward_std": 0.08100546710193157, | |
| "rewards/length_reward": 0.07098214700818062, | |
| "rewards/similarity_reward": 0.6386776566505432, | |
| "step": 318 | |
| }, | |
| { | |
| "completion_length": 248.56697845458984, | |
| "epoch": 0.8506666666666667, | |
| "grad_norm": 0.6967301959645911, | |
| "kl": 0.004791259765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0788, | |
| "reward": 0.712384045124054, | |
| "reward_std": 0.08104405552148819, | |
| "rewards/length_reward": 0.07633929327130318, | |
| "rewards/similarity_reward": 0.6360447704792023, | |
| "step": 319 | |
| }, | |
| { | |
| "completion_length": 253.03126525878906, | |
| "epoch": 0.8533333333333334, | |
| "grad_norm": 0.6240879358111411, | |
| "kl": 0.0043487548828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0052, | |
| "reward": 0.6519544422626495, | |
| "reward_std": 0.0803583487868309, | |
| "rewards/length_reward": 0.0691964440047741, | |
| "rewards/similarity_reward": 0.5827580094337463, | |
| "step": 320 | |
| }, | |
| { | |
| "completion_length": 242.95091247558594, | |
| "epoch": 0.856, | |
| "grad_norm": 0.549313108944523, | |
| "kl": 0.0028839111328125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0208, | |
| "reward": 0.7446077167987823, | |
| "reward_std": 0.09269357472658157, | |
| "rewards/length_reward": 0.07410714775323868, | |
| "rewards/similarity_reward": 0.6705006062984467, | |
| "step": 321 | |
| }, | |
| { | |
| "completion_length": 224.89286041259766, | |
| "epoch": 0.8586666666666667, | |
| "grad_norm": 0.7316053385982602, | |
| "kl": 0.00386810302734375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0086, | |
| "reward": 0.6810529232025146, | |
| "reward_std": 0.06378122419118881, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.6096242964267731, | |
| "step": 322 | |
| }, | |
| { | |
| "completion_length": 236.2009048461914, | |
| "epoch": 0.8613333333333333, | |
| "grad_norm": 0.6760501533209452, | |
| "kl": 0.0053558349609375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0282, | |
| "reward": 0.6928089559078217, | |
| "reward_std": 0.07963042706251144, | |
| "rewards/length_reward": 0.07410715147852898, | |
| "rewards/similarity_reward": 0.6187017858028412, | |
| "step": 323 | |
| }, | |
| { | |
| "completion_length": 241.27233123779297, | |
| "epoch": 0.864, | |
| "grad_norm": 0.7174681952375248, | |
| "kl": 0.0048065185546875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0184, | |
| "reward": 0.7107245922088623, | |
| "reward_std": 0.07196836359798908, | |
| "rewards/length_reward": 0.07321429252624512, | |
| "rewards/similarity_reward": 0.6375102698802948, | |
| "step": 324 | |
| }, | |
| { | |
| "completion_length": 219.32144165039062, | |
| "epoch": 0.8666666666666667, | |
| "grad_norm": 0.6619044506684808, | |
| "kl": 0.005645751953125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0104, | |
| "reward": 0.6078022122383118, | |
| "reward_std": 0.09693693742156029, | |
| "rewards/length_reward": 0.07098215073347092, | |
| "rewards/similarity_reward": 0.5368200242519379, | |
| "step": 325 | |
| }, | |
| { | |
| "completion_length": 228.84375762939453, | |
| "epoch": 0.8693333333333333, | |
| "grad_norm": 0.6998337254787387, | |
| "kl": 0.00328826904296875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0045, | |
| "reward": 0.7553087770938873, | |
| "reward_std": 0.07146935909986496, | |
| "rewards/length_reward": 0.0736607275903225, | |
| "rewards/similarity_reward": 0.6816481053829193, | |
| "step": 326 | |
| }, | |
| { | |
| "completion_length": 288.2901916503906, | |
| "epoch": 0.872, | |
| "grad_norm": 0.6156275180911798, | |
| "kl": 0.00383758544921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0138, | |
| "reward": 0.7228290438652039, | |
| "reward_std": 0.08385154232382774, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.6514004170894623, | |
| "step": 327 | |
| }, | |
| { | |
| "completion_length": 260.9821472167969, | |
| "epoch": 0.8746666666666667, | |
| "grad_norm": 0.6463359471511301, | |
| "kl": 0.0041351318359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0624, | |
| "reward": 0.7054339051246643, | |
| "reward_std": 0.08808758854866028, | |
| "rewards/length_reward": 0.06339286640286446, | |
| "rewards/similarity_reward": 0.6420409977436066, | |
| "step": 328 | |
| }, | |
| { | |
| "completion_length": 247.34376525878906, | |
| "epoch": 0.8773333333333333, | |
| "grad_norm": 0.7642501038807545, | |
| "kl": 0.00417327880859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0269, | |
| "reward": 0.6869321167469025, | |
| "reward_std": 0.06861887872219086, | |
| "rewards/length_reward": 0.06741072982549667, | |
| "rewards/similarity_reward": 0.61952143907547, | |
| "step": 329 | |
| }, | |
| { | |
| "completion_length": 267.3928756713867, | |
| "epoch": 0.88, | |
| "grad_norm": 0.6571448003835118, | |
| "kl": 0.0063934326171875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0592, | |
| "reward": 0.6967031061649323, | |
| "reward_std": 0.08689341694116592, | |
| "rewards/length_reward": 0.07455357909202576, | |
| "rewards/similarity_reward": 0.6221494972705841, | |
| "step": 330 | |
| }, | |
| { | |
| "completion_length": 251.84822845458984, | |
| "epoch": 0.8826666666666667, | |
| "grad_norm": 0.6160126783055467, | |
| "kl": 0.00362396240234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0047, | |
| "reward": 0.6543110907077789, | |
| "reward_std": 0.08520649373531342, | |
| "rewards/length_reward": 0.06562501192092896, | |
| "rewards/similarity_reward": 0.58868607878685, | |
| "step": 331 | |
| }, | |
| { | |
| "completion_length": 277.6384048461914, | |
| "epoch": 0.8853333333333333, | |
| "grad_norm": 0.6951712149387916, | |
| "kl": 0.00399017333984375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0197, | |
| "reward": 0.6851289868354797, | |
| "reward_std": 0.09784185141324997, | |
| "rewards/length_reward": 0.07232143357396126, | |
| "rewards/similarity_reward": 0.6128075420856476, | |
| "step": 332 | |
| }, | |
| { | |
| "completion_length": 256.7366180419922, | |
| "epoch": 0.888, | |
| "grad_norm": 0.5992862799755034, | |
| "kl": 0.004791259765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0491, | |
| "reward": 0.6802008152008057, | |
| "reward_std": 0.09179921820759773, | |
| "rewards/length_reward": 0.07008929178118706, | |
| "rewards/similarity_reward": 0.6101114749908447, | |
| "step": 333 | |
| }, | |
| { | |
| "completion_length": 215.21430206298828, | |
| "epoch": 0.8906666666666667, | |
| "grad_norm": 0.709856632058597, | |
| "kl": 0.0052032470703125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0124, | |
| "reward": 0.692722350358963, | |
| "reward_std": 0.08042297512292862, | |
| "rewards/length_reward": 0.060714298859238625, | |
| "rewards/similarity_reward": 0.6320080757141113, | |
| "step": 334 | |
| }, | |
| { | |
| "completion_length": 244.75001525878906, | |
| "epoch": 0.8933333333333333, | |
| "grad_norm": 0.623193882721459, | |
| "kl": 0.00424957275390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0358, | |
| "reward": 0.7010281980037689, | |
| "reward_std": 0.08354394137859344, | |
| "rewards/length_reward": 0.07098215445876122, | |
| "rewards/similarity_reward": 0.630046010017395, | |
| "step": 335 | |
| }, | |
| { | |
| "completion_length": 222.87500762939453, | |
| "epoch": 0.896, | |
| "grad_norm": 0.7264124980613719, | |
| "kl": 0.0045928955078125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0194, | |
| "reward": 0.6454309821128845, | |
| "reward_std": 0.10383091494441032, | |
| "rewards/length_reward": 0.06562501192092896, | |
| "rewards/similarity_reward": 0.579805999994278, | |
| "step": 336 | |
| }, | |
| { | |
| "completion_length": 260.1696548461914, | |
| "epoch": 0.8986666666666666, | |
| "grad_norm": 0.5907324344548502, | |
| "kl": 0.0032806396484375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0162, | |
| "reward": 0.6906337440013885, | |
| "reward_std": 0.06872132420539856, | |
| "rewards/length_reward": 0.07723214477300644, | |
| "rewards/similarity_reward": 0.6134015619754791, | |
| "step": 337 | |
| }, | |
| { | |
| "completion_length": 245.22322845458984, | |
| "epoch": 0.9013333333333333, | |
| "grad_norm": 0.6126450019945038, | |
| "kl": 0.00359344482421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0172, | |
| "reward": 0.7013856172561646, | |
| "reward_std": 0.07948705554008484, | |
| "rewards/length_reward": 0.07678572088479996, | |
| "rewards/similarity_reward": 0.6245998740196228, | |
| "step": 338 | |
| }, | |
| { | |
| "completion_length": 250.85269927978516, | |
| "epoch": 0.904, | |
| "grad_norm": 0.7156180383403373, | |
| "kl": 0.00537109375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0313, | |
| "reward": 0.6530185341835022, | |
| "reward_std": 0.0950470082461834, | |
| "rewards/length_reward": 0.07187500596046448, | |
| "rewards/similarity_reward": 0.5811434835195541, | |
| "step": 339 | |
| }, | |
| { | |
| "completion_length": 253.91964721679688, | |
| "epoch": 0.9066666666666666, | |
| "grad_norm": 0.5742388325200097, | |
| "kl": 0.004302978515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.01, | |
| "reward": 0.6900386214256287, | |
| "reward_std": 0.06002875231206417, | |
| "rewards/length_reward": 0.07991071790456772, | |
| "rewards/similarity_reward": 0.6101278960704803, | |
| "step": 340 | |
| }, | |
| { | |
| "completion_length": 214.0446548461914, | |
| "epoch": 0.9093333333333333, | |
| "grad_norm": 0.7992075593440748, | |
| "kl": 0.0047760009765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.017, | |
| "reward": 0.6177321374416351, | |
| "reward_std": 0.0978124737739563, | |
| "rewards/length_reward": 0.058482153341174126, | |
| "rewards/similarity_reward": 0.559249997138977, | |
| "step": 341 | |
| }, | |
| { | |
| "completion_length": 244.4732208251953, | |
| "epoch": 0.912, | |
| "grad_norm": 0.8258653848140411, | |
| "kl": 0.0045623779296875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0631, | |
| "reward": 0.6902224123477936, | |
| "reward_std": 0.08696568384766579, | |
| "rewards/length_reward": 0.061607154086232185, | |
| "rewards/similarity_reward": 0.6286152601242065, | |
| "step": 342 | |
| }, | |
| { | |
| "completion_length": 217.93304443359375, | |
| "epoch": 0.9146666666666666, | |
| "grad_norm": 0.6997396550994213, | |
| "kl": 0.005523681640625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0019, | |
| "reward": 0.6602669954299927, | |
| "reward_std": 0.09806296601891518, | |
| "rewards/length_reward": 0.06428572349250317, | |
| "rewards/similarity_reward": 0.5959812104701996, | |
| "step": 343 | |
| }, | |
| { | |
| "completion_length": 285.45538330078125, | |
| "epoch": 0.9173333333333333, | |
| "grad_norm": 0.6020909063855457, | |
| "kl": 0.00421142578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0088, | |
| "reward": 0.6885746419429779, | |
| "reward_std": 0.09227034822106361, | |
| "rewards/length_reward": 0.07232143729925156, | |
| "rewards/similarity_reward": 0.6162531673908234, | |
| "step": 344 | |
| }, | |
| { | |
| "completion_length": 260.15179443359375, | |
| "epoch": 0.92, | |
| "grad_norm": 0.8278574941445127, | |
| "kl": 0.0059051513671875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1217, | |
| "reward": 0.6081913709640503, | |
| "reward_std": 0.18488339334726334, | |
| "rewards/length_reward": 0.058482155203819275, | |
| "rewards/similarity_reward": 0.5497092008590698, | |
| "step": 345 | |
| }, | |
| { | |
| "completion_length": 272.45091247558594, | |
| "epoch": 0.9226666666666666, | |
| "grad_norm": 0.5703333877662986, | |
| "kl": 0.00386810302734375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.021, | |
| "reward": 0.7158257365226746, | |
| "reward_std": 0.08523573726415634, | |
| "rewards/length_reward": 0.07991071790456772, | |
| "rewards/similarity_reward": 0.6359150111675262, | |
| "step": 346 | |
| }, | |
| { | |
| "completion_length": 291.5714416503906, | |
| "epoch": 0.9253333333333333, | |
| "grad_norm": 0.664603610390973, | |
| "kl": 0.00350189208984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0368, | |
| "reward": 0.7160574495792389, | |
| "reward_std": 0.0684958528727293, | |
| "rewards/length_reward": 0.06651787087321281, | |
| "rewards/similarity_reward": 0.6495395302772522, | |
| "step": 347 | |
| }, | |
| { | |
| "completion_length": 301.0357360839844, | |
| "epoch": 0.928, | |
| "grad_norm": 0.5288343523218547, | |
| "kl": 0.00455474853515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.015, | |
| "reward": 0.7117947041988373, | |
| "reward_std": 0.10228880494832993, | |
| "rewards/length_reward": 0.06741072237491608, | |
| "rewards/similarity_reward": 0.6443840265274048, | |
| "step": 348 | |
| }, | |
| { | |
| "completion_length": 260.44197845458984, | |
| "epoch": 0.9306666666666666, | |
| "grad_norm": 0.6372539241149925, | |
| "kl": 0.00432586669921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.031, | |
| "reward": 0.7437091171741486, | |
| "reward_std": 0.07142951153218746, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.6745127141475677, | |
| "step": 349 | |
| }, | |
| { | |
| "completion_length": 266.7232208251953, | |
| "epoch": 0.9333333333333333, | |
| "grad_norm": 0.7417544957869086, | |
| "kl": 0.003936767578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0914, | |
| "reward": 0.7335431277751923, | |
| "reward_std": 0.10938547924160957, | |
| "rewards/length_reward": 0.07321429252624512, | |
| "rewards/similarity_reward": 0.6603288650512695, | |
| "step": 350 | |
| }, | |
| { | |
| "completion_length": 263.49554443359375, | |
| "epoch": 0.936, | |
| "grad_norm": 0.7076854337089484, | |
| "kl": 0.008270263671875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0148, | |
| "reward": 0.6161931157112122, | |
| "reward_std": 0.12820692360401154, | |
| "rewards/length_reward": 0.06339287385344505, | |
| "rewards/similarity_reward": 0.5528002381324768, | |
| "step": 351 | |
| }, | |
| { | |
| "completion_length": 263.9866180419922, | |
| "epoch": 0.9386666666666666, | |
| "grad_norm": 0.5813263791328529, | |
| "kl": 0.00344085693359375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0071, | |
| "reward": 0.6747315526008606, | |
| "reward_std": 0.09237341955304146, | |
| "rewards/length_reward": 0.06607143953442574, | |
| "rewards/similarity_reward": 0.6086601614952087, | |
| "step": 352 | |
| }, | |
| { | |
| "completion_length": 242.46875762939453, | |
| "epoch": 0.9413333333333334, | |
| "grad_norm": 0.6091815171465548, | |
| "kl": 0.0047607421875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0248, | |
| "reward": 0.6233691573143005, | |
| "reward_std": 0.12349110096693039, | |
| "rewards/length_reward": 0.0647321529686451, | |
| "rewards/similarity_reward": 0.5586370527744293, | |
| "step": 353 | |
| }, | |
| { | |
| "completion_length": 265.0089416503906, | |
| "epoch": 0.944, | |
| "grad_norm": 0.5824155704971612, | |
| "kl": 0.00321197509765625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0139, | |
| "reward": 0.7225279808044434, | |
| "reward_std": 0.0762885119765997, | |
| "rewards/length_reward": 0.0714285857975483, | |
| "rewards/similarity_reward": 0.651099443435669, | |
| "step": 354 | |
| }, | |
| { | |
| "completion_length": 276.56697845458984, | |
| "epoch": 0.9466666666666667, | |
| "grad_norm": 0.5410198221272389, | |
| "kl": 0.0042724609375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0064, | |
| "reward": 0.7190838754177094, | |
| "reward_std": 0.11137175559997559, | |
| "rewards/length_reward": 0.07544643431901932, | |
| "rewards/similarity_reward": 0.6436374485492706, | |
| "step": 355 | |
| }, | |
| { | |
| "completion_length": 299.20982360839844, | |
| "epoch": 0.9493333333333334, | |
| "grad_norm": 0.5828251751367955, | |
| "kl": 0.0042877197265625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0076, | |
| "reward": 0.7062688171863556, | |
| "reward_std": 0.0902559757232666, | |
| "rewards/length_reward": 0.0781250037252903, | |
| "rewards/similarity_reward": 0.6281438171863556, | |
| "step": 356 | |
| }, | |
| { | |
| "completion_length": 261.5714416503906, | |
| "epoch": 0.952, | |
| "grad_norm": 0.662401613193247, | |
| "kl": 0.00417327880859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.1072, | |
| "reward": 0.6910728216171265, | |
| "reward_std": 0.12025736272335052, | |
| "rewards/length_reward": 0.06517858058214188, | |
| "rewards/similarity_reward": 0.6258941888809204, | |
| "step": 357 | |
| }, | |
| { | |
| "completion_length": 273.22322845458984, | |
| "epoch": 0.9546666666666667, | |
| "grad_norm": 0.5897487229990825, | |
| "kl": 0.0035400390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0006, | |
| "reward": 0.7075876295566559, | |
| "reward_std": 0.06886249966919422, | |
| "rewards/length_reward": 0.07098215445876122, | |
| "rewards/similarity_reward": 0.6366054713726044, | |
| "step": 358 | |
| }, | |
| { | |
| "completion_length": 287.7634048461914, | |
| "epoch": 0.9573333333333334, | |
| "grad_norm": 0.5282572684933031, | |
| "kl": 0.00276947021484375, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0016, | |
| "reward": 0.7663991749286652, | |
| "reward_std": 0.08505138382315636, | |
| "rewards/length_reward": 0.07767857611179352, | |
| "rewards/similarity_reward": 0.6887206435203552, | |
| "step": 359 | |
| }, | |
| { | |
| "completion_length": 245.16519165039062, | |
| "epoch": 0.96, | |
| "grad_norm": 0.6891670466809943, | |
| "kl": 0.00555419921875, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0045, | |
| "reward": 0.6559944748878479, | |
| "reward_std": 0.10601956769824028, | |
| "rewards/length_reward": 0.0691964402794838, | |
| "rewards/similarity_reward": 0.5867980420589447, | |
| "step": 360 | |
| }, | |
| { | |
| "completion_length": 270.0357208251953, | |
| "epoch": 0.9626666666666667, | |
| "grad_norm": 0.5428726789809684, | |
| "kl": 0.0041961669921875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0106, | |
| "reward": 0.6817942559719086, | |
| "reward_std": 0.07338645495474339, | |
| "rewards/length_reward": 0.07767857611179352, | |
| "rewards/similarity_reward": 0.6041156351566315, | |
| "step": 361 | |
| }, | |
| { | |
| "completion_length": 243.84376525878906, | |
| "epoch": 0.9653333333333334, | |
| "grad_norm": 0.643839801785035, | |
| "kl": 0.0121307373046875, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0103, | |
| "reward": 0.6772570908069611, | |
| "reward_std": 0.06569460779428482, | |
| "rewards/length_reward": 0.06964286789298058, | |
| "rewards/similarity_reward": 0.6076142191886902, | |
| "step": 362 | |
| }, | |
| { | |
| "completion_length": 242.50894165039062, | |
| "epoch": 0.968, | |
| "grad_norm": 0.5946163970209724, | |
| "kl": 0.00385284423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0095, | |
| "reward": 0.684212863445282, | |
| "reward_std": 0.09431854262948036, | |
| "rewards/length_reward": 0.07455357909202576, | |
| "rewards/similarity_reward": 0.6096592247486115, | |
| "step": 363 | |
| }, | |
| { | |
| "completion_length": 266.8080520629883, | |
| "epoch": 0.9706666666666667, | |
| "grad_norm": 0.5952393619644073, | |
| "kl": 0.0045318603515625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0054, | |
| "reward": 0.6641288101673126, | |
| "reward_std": 0.08274504169821739, | |
| "rewards/length_reward": 0.06651787459850311, | |
| "rewards/similarity_reward": 0.5976109206676483, | |
| "step": 364 | |
| }, | |
| { | |
| "completion_length": 280.81251525878906, | |
| "epoch": 0.9733333333333334, | |
| "grad_norm": 0.704408476901472, | |
| "kl": 0.006378173828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0302, | |
| "reward": 0.6689063608646393, | |
| "reward_std": 0.06241089664399624, | |
| "rewards/length_reward": 0.06964286789298058, | |
| "rewards/similarity_reward": 0.5992635488510132, | |
| "step": 365 | |
| }, | |
| { | |
| "completion_length": 272.53126525878906, | |
| "epoch": 0.976, | |
| "grad_norm": 0.6740721279945435, | |
| "kl": 0.005615234375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0723, | |
| "reward": 0.6874203383922577, | |
| "reward_std": 0.08594712242484093, | |
| "rewards/length_reward": 0.06785715371370316, | |
| "rewards/similarity_reward": 0.6195632219314575, | |
| "step": 366 | |
| }, | |
| { | |
| "completion_length": 237.32144165039062, | |
| "epoch": 0.9786666666666667, | |
| "grad_norm": 0.6453259649723537, | |
| "kl": 0.0041046142578125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0057, | |
| "reward": 0.6572641432285309, | |
| "reward_std": 0.07327758520841599, | |
| "rewards/length_reward": 0.07053572125732899, | |
| "rewards/similarity_reward": 0.5867283642292023, | |
| "step": 367 | |
| }, | |
| { | |
| "completion_length": 230.68304443359375, | |
| "epoch": 0.9813333333333333, | |
| "grad_norm": 0.6672094875832669, | |
| "kl": 0.00567626953125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0058, | |
| "reward": 0.6544754207134247, | |
| "reward_std": 0.09319397434592247, | |
| "rewards/length_reward": 0.071428582072258, | |
| "rewards/similarity_reward": 0.5830467939376831, | |
| "step": 368 | |
| }, | |
| { | |
| "completion_length": 183.78125762939453, | |
| "epoch": 0.984, | |
| "grad_norm": 0.7366023492253149, | |
| "kl": 0.00495147705078125, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0059, | |
| "reward": 0.6258625388145447, | |
| "reward_std": 0.08410684764385223, | |
| "rewards/length_reward": 0.07187500968575478, | |
| "rewards/similarity_reward": 0.5539875626564026, | |
| "step": 369 | |
| }, | |
| { | |
| "completion_length": 242.49555206298828, | |
| "epoch": 0.9866666666666667, | |
| "grad_norm": 0.6621715728429993, | |
| "kl": 0.0055999755859375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0227, | |
| "reward": 0.6772402822971344, | |
| "reward_std": 0.08659335412085056, | |
| "rewards/length_reward": 0.0714285783469677, | |
| "rewards/similarity_reward": 0.6058117151260376, | |
| "step": 370 | |
| }, | |
| { | |
| "completion_length": 223.2366180419922, | |
| "epoch": 0.9893333333333333, | |
| "grad_norm": 0.7315442476679772, | |
| "kl": 0.0076751708984375, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0034, | |
| "reward": 0.6550423204898834, | |
| "reward_std": 0.08679429814219475, | |
| "rewards/length_reward": 0.06830358132719994, | |
| "rewards/similarity_reward": 0.5867387652397156, | |
| "step": 371 | |
| }, | |
| { | |
| "completion_length": 279.18751525878906, | |
| "epoch": 0.992, | |
| "grad_norm": 0.7043801053601064, | |
| "kl": 0.005035400390625, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0732, | |
| "reward": 0.6848048269748688, | |
| "reward_std": 0.10013966262340546, | |
| "rewards/length_reward": 0.07991071417927742, | |
| "rewards/similarity_reward": 0.6048941314220428, | |
| "step": 372 | |
| }, | |
| { | |
| "completion_length": 221.90625762939453, | |
| "epoch": 0.9946666666666667, | |
| "grad_norm": 0.6692794616147932, | |
| "kl": 0.0042877197265625, | |
| "learning_rate": 2e-06, | |
| "loss": -0.0054, | |
| "reward": 0.6771494746208191, | |
| "reward_std": 0.0797378458082676, | |
| "rewards/length_reward": 0.061607155948877335, | |
| "rewards/similarity_reward": 0.6155422627925873, | |
| "step": 373 | |
| }, | |
| { | |
| "completion_length": 287.4330596923828, | |
| "epoch": 0.9973333333333333, | |
| "grad_norm": 0.6988535931211539, | |
| "kl": 0.0048828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0135, | |
| "reward": 0.7209034264087677, | |
| "reward_std": 0.0916130281984806, | |
| "rewards/length_reward": 0.07321429252624512, | |
| "rewards/similarity_reward": 0.647689163684845, | |
| "step": 374 | |
| }, | |
| { | |
| "completion_length": 234.6508026123047, | |
| "epoch": 1.0, | |
| "grad_norm": 0.7155440816719996, | |
| "kl": 0.0072784423828125, | |
| "learning_rate": 2e-06, | |
| "loss": 0.0237, | |
| "reward": 0.6349575817584991, | |
| "reward_std": 0.13314566388726234, | |
| "rewards/length_reward": 0.056696439161896706, | |
| "rewards/similarity_reward": 0.578261137008667, | |
| "step": 375 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 375, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |