File size: 12,695 Bytes
e7312f1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.9473684210526314,
"eval_steps": 10,
"global_step": 21,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.14035087719298245,
"grad_norm": 36.83965274682509,
"learning_rate": 3.3333333333333333e-06,
"logits/chosen": -0.6751565337181091,
"logits/rejected": -0.680110514163971,
"logps/chosen": -52.487876892089844,
"logps/rejected": -58.423255920410156,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.2807017543859649,
"grad_norm": 36.26353717517849,
"learning_rate": 6.666666666666667e-06,
"logits/chosen": -0.7261925339698792,
"logits/rejected": -0.7052676677703857,
"logps/chosen": -51.76400375366211,
"logps/rejected": -58.045860290527344,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 2
},
{
"epoch": 0.42105263157894735,
"grad_norm": 6.35145459559084,
"learning_rate": 1e-05,
"logits/chosen": -0.6268625259399414,
"logits/rejected": -0.44894033670425415,
"logps/chosen": -42.51791763305664,
"logps/rejected": -77.13123321533203,
"loss": 0.1248,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.8370370864868164,
"rewards/margins": 2.879814386367798,
"rewards/rejected": -2.0427772998809814,
"step": 3
},
{
"epoch": 0.5614035087719298,
"grad_norm": 6.045707313215622,
"learning_rate": 9.924038765061042e-06,
"logits/chosen": -0.49778643250465393,
"logits/rejected": -0.3794001340866089,
"logps/chosen": -40.093902587890625,
"logps/rejected": -100.6180191040039,
"loss": 0.0342,
"rewards/accuracies": 1.0,
"rewards/chosen": 1.0513410568237305,
"rewards/margins": 5.476510047912598,
"rewards/rejected": -4.425168514251709,
"step": 4
},
{
"epoch": 0.7017543859649122,
"grad_norm": 3.5732460989636903,
"learning_rate": 9.698463103929542e-06,
"logits/chosen": -0.1468430608510971,
"logits/rejected": -0.014196997508406639,
"logps/chosen": -47.64195251464844,
"logps/rejected": -108.50719451904297,
"loss": 0.035,
"rewards/accuracies": 0.96875,
"rewards/chosen": 0.4784576892852783,
"rewards/margins": 5.548664093017578,
"rewards/rejected": -5.070206165313721,
"step": 5
},
{
"epoch": 0.8421052631578947,
"grad_norm": 5.73507612352672,
"learning_rate": 9.330127018922195e-06,
"logits/chosen": -0.45869290828704834,
"logits/rejected": -0.35349956154823303,
"logps/chosen": -23.860214233398438,
"logps/rejected": -72.7856216430664,
"loss": 0.0381,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.7691407203674316,
"rewards/margins": 4.281482696533203,
"rewards/rejected": -1.512341856956482,
"step": 6
},
{
"epoch": 0.9824561403508771,
"grad_norm": 0.20133266492379556,
"learning_rate": 8.83022221559489e-06,
"logits/chosen": 0.02529796212911606,
"logits/rejected": -0.027504097670316696,
"logps/chosen": -20.159576416015625,
"logps/rejected": -136.86343383789062,
"loss": 0.0021,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.0319323539733887,
"rewards/margins": 10.98936653137207,
"rewards/rejected": -7.957433700561523,
"step": 7
},
{
"epoch": 1.1228070175438596,
"grad_norm": 1.538785447893909,
"learning_rate": 8.213938048432697e-06,
"logits/chosen": -0.09107446670532227,
"logits/rejected": -0.030364712700247765,
"logps/chosen": -31.587915420532227,
"logps/rejected": -148.98675537109375,
"loss": 0.0988,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.1359970569610596,
"rewards/margins": 11.210424423217773,
"rewards/rejected": -9.074427604675293,
"step": 8
},
{
"epoch": 1.263157894736842,
"grad_norm": 2.484440900292177,
"learning_rate": 7.500000000000001e-06,
"logits/chosen": -0.021966181695461273,
"logits/rejected": -0.035571545362472534,
"logps/chosen": -22.42191505432129,
"logps/rejected": -139.77471923828125,
"loss": 0.0482,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.9303808212280273,
"rewards/margins": 11.172625541687012,
"rewards/rejected": -8.242246627807617,
"step": 9
},
{
"epoch": 1.4035087719298245,
"grad_norm": 2.063248053979488,
"learning_rate": 6.710100716628345e-06,
"logits/chosen": -0.1402013897895813,
"logits/rejected": -0.09476425498723984,
"logps/chosen": -22.920854568481445,
"logps/rejected": -139.89112854003906,
"loss": 0.1011,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.8295440673828125,
"rewards/margins": 11.169378280639648,
"rewards/rejected": -8.339835166931152,
"step": 10
},
{
"epoch": 1.4035087719298245,
"eval_logits/chosen": -0.5006358027458191,
"eval_logits/rejected": -0.5087898373603821,
"eval_logps/chosen": -21.234792709350586,
"eval_logps/rejected": -123.04338073730469,
"eval_loss": 0.07855287194252014,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 3.130305767059326,
"eval_rewards/margins": 9.576081275939941,
"eval_rewards/rejected": -6.445775032043457,
"eval_runtime": 16.4971,
"eval_samples_per_second": 6.062,
"eval_steps_per_second": 3.031,
"step": 10
},
{
"epoch": 1.543859649122807,
"grad_norm": 2.01867368401983,
"learning_rate": 5.8682408883346535e-06,
"logits/chosen": -0.3766348659992218,
"logits/rejected": -0.22392578423023224,
"logps/chosen": -23.370777130126953,
"logps/rejected": -121.3629150390625,
"loss": 0.0607,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.7753195762634277,
"rewards/margins": 9.217280387878418,
"rewards/rejected": -6.441961765289307,
"step": 11
},
{
"epoch": 1.6842105263157894,
"grad_norm": 4.079342276288671,
"learning_rate": 5e-06,
"logits/chosen": -0.5493069887161255,
"logits/rejected": -0.3390986919403076,
"logps/chosen": -26.047971725463867,
"logps/rejected": -90.44429016113281,
"loss": 0.0339,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.553872585296631,
"rewards/margins": 5.881150722503662,
"rewards/rejected": -3.3272786140441895,
"step": 12
},
{
"epoch": 1.8245614035087718,
"grad_norm": 1.7556159148296524,
"learning_rate": 4.131759111665349e-06,
"logits/chosen": -0.4146791398525238,
"logits/rejected": -0.286813884973526,
"logps/chosen": -24.490224838256836,
"logps/rejected": -102.24156188964844,
"loss": 0.0562,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.6847214698791504,
"rewards/margins": 7.214890003204346,
"rewards/rejected": -4.5301690101623535,
"step": 13
},
{
"epoch": 1.9649122807017543,
"grad_norm": 1.5851709952279243,
"learning_rate": 3.289899283371657e-06,
"logits/chosen": -0.44919806718826294,
"logits/rejected": -0.38574808835983276,
"logps/chosen": -21.76868438720703,
"logps/rejected": -93.4135513305664,
"loss": 0.0404,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.8682363033294678,
"rewards/margins": 6.415185928344727,
"rewards/rejected": -3.5469493865966797,
"step": 14
},
{
"epoch": 2.1052631578947367,
"grad_norm": 0.8663363353210317,
"learning_rate": 2.5000000000000015e-06,
"logits/chosen": -0.3753425180912018,
"logits/rejected": -0.3148278295993805,
"logps/chosen": -27.85059928894043,
"logps/rejected": -96.494140625,
"loss": 0.0092,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.4423599243164062,
"rewards/margins": 6.41550874710083,
"rewards/rejected": -3.9731483459472656,
"step": 15
},
{
"epoch": 2.245614035087719,
"grad_norm": 0.46918563058013124,
"learning_rate": 1.7860619515673034e-06,
"logits/chosen": -0.37954726815223694,
"logits/rejected": -0.3316181004047394,
"logps/chosen": -25.504596710205078,
"logps/rejected": -102.33647155761719,
"loss": 0.0106,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.5688281059265137,
"rewards/margins": 7.0828657150268555,
"rewards/rejected": -4.514037132263184,
"step": 16
},
{
"epoch": 2.3859649122807016,
"grad_norm": 1.3915012816414571,
"learning_rate": 1.1697777844051105e-06,
"logits/chosen": -0.32976603507995605,
"logits/rejected": -0.24117198586463928,
"logps/chosen": -22.648927688598633,
"logps/rejected": -111.1461181640625,
"loss": 0.0722,
"rewards/accuracies": 0.984375,
"rewards/chosen": 2.808568239212036,
"rewards/margins": 8.266373634338379,
"rewards/rejected": -5.457805633544922,
"step": 17
},
{
"epoch": 2.526315789473684,
"grad_norm": 0.7864294623647854,
"learning_rate": 6.698729810778065e-07,
"logits/chosen": -0.31580400466918945,
"logits/rejected": -0.23523080348968506,
"logps/chosen": -21.38547706604004,
"logps/rejected": -112.5594482421875,
"loss": 0.0132,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.9571423530578613,
"rewards/margins": 8.440750122070312,
"rewards/rejected": -5.483607769012451,
"step": 18
},
{
"epoch": 2.6666666666666665,
"grad_norm": 1.6842586899427825,
"learning_rate": 3.015368960704584e-07,
"logits/chosen": -0.227472722530365,
"logits/rejected": -0.19328871369361877,
"logps/chosen": -18.14565086364746,
"logps/rejected": -115.78607177734375,
"loss": 0.0465,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.302818536758423,
"rewards/margins": 9.093223571777344,
"rewards/rejected": -5.790404796600342,
"step": 19
},
{
"epoch": 2.807017543859649,
"grad_norm": 0.37240158397575107,
"learning_rate": 7.59612349389599e-08,
"logits/chosen": -0.28918278217315674,
"logits/rejected": -0.20876720547676086,
"logps/chosen": -20.228591918945312,
"logps/rejected": -110.49909210205078,
"loss": 0.0085,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.1952502727508545,
"rewards/margins": 8.55958366394043,
"rewards/rejected": -5.36433219909668,
"step": 20
},
{
"epoch": 2.807017543859649,
"eval_logits/chosen": -0.5716415643692017,
"eval_logits/rejected": -0.5896289944648743,
"eval_logps/chosen": -19.139745712280273,
"eval_logps/rejected": -113.21296691894531,
"eval_loss": 0.07458853721618652,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 3.339810371398926,
"eval_rewards/margins": 8.802544593811035,
"eval_rewards/rejected": -5.462734222412109,
"eval_runtime": 18.3879,
"eval_samples_per_second": 5.438,
"eval_steps_per_second": 2.719,
"step": 20
},
{
"epoch": 2.9473684210526314,
"grad_norm": 2.3675721766606554,
"learning_rate": 0.0,
"logits/chosen": -0.3103935122489929,
"logits/rejected": -0.23779892921447754,
"logps/chosen": -24.52212905883789,
"logps/rejected": -107.23027038574219,
"loss": 0.0789,
"rewards/accuracies": 0.96875,
"rewards/chosen": 2.7476420402526855,
"rewards/margins": 7.737358093261719,
"rewards/rejected": -4.989716529846191,
"step": 21
}
],
"logging_steps": 1,
"max_steps": 21,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 5,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 1322411360256.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|